<!DOCTYPE html>
<html>
<head><meta charset="utf-8" />

<title>32.transformer_colab</title>

<script src="https://cdnjs.cloudflare.com/ajax/libs/require.js/2.1.10/require.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/2.0.3/jquery.min.js"></script>

<script>
(function() {
  function addWidgetsRenderer() {
    var mimeElement = document.querySelector('script[type="application/vnd.jupyter.widget-view+json"]');
    var scriptElement = document.createElement('script');
    var widgetRendererSrc = '@jupyter-widgets/html-manager@*/dist/embed-amd.js';
    var widgetState;

    // Fallback for older version:
    try {
      widgetState = mimeElement && JSON.parse(mimeElement.innerHTML);

      if (widgetState && (widgetState.version_major < 2 || !widgetState.version_major)) {
        widgetRendererSrc = 'jupyter-js-widgets@*/dist/embed.js';
      }
    } catch(e) {}

    scriptElement.src = widgetRendererSrc;
    document.body.appendChild(scriptElement);
  }

  document.addEventListener('DOMContentLoaded', addWidgetsRenderer);
}());
</script>

<style type="text/css">
    /*!
*
* Twitter Bootstrap
*
*/
/*!
 * Bootstrap v3.3.7 (http://getbootstrap.com)
 * Copyright 2011-2016 Twitter, Inc.
 * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE)
 */
/*! normalize.css v3.0.3 | MIT License | github.com/necolas/normalize.css */
html {
  font-family: sans-serif;
  -ms-text-size-adjust: 100%;
  -webkit-text-size-adjust: 100%;
}
body {
  margin: 0;
}
article,
aside,
details,
figcaption,
figure,
footer,
header,
hgroup,
main,
menu,
nav,
section,
summary {
  display: block;
}
audio,
canvas,
progress,
video {
  display: inline-block;
  vertical-align: baseline;
}
audio:not([controls]) {
  display: none;
  height: 0;
}
[hidden],
template {
  display: none;
}
a {
  background-color: transparent;
}
a:active,
a:hover {
  outline: 0;
}
abbr[title] {
  border-bottom: 1px dotted;
}
b,
strong {
  font-weight: bold;
}
dfn {
  font-style: italic;
}
h1 {
  font-size: 2em;
  margin: 0.67em 0;
}
mark {
  background: #ff0;
  color: #000;
}
small {
  font-size: 80%;
}
sub,
sup {
  font-size: 75%;
  line-height: 0;
  position: relative;
  vertical-align: baseline;
}
sup {
  top: -0.5em;
}
sub {
  bottom: -0.25em;
}
img {
  border: 0;
}
svg:not(:root) {
  overflow: hidden;
}
figure {
  margin: 1em 40px;
}
hr {
  box-sizing: content-box;
  height: 0;
}
pre {
  overflow: auto;
}
code,
kbd,
pre,
samp {
  font-family: monospace, monospace;
  font-size: 1em;
}
button,
input,
optgroup,
select,
textarea {
  color: inherit;
  font: inherit;
  margin: 0;
}
button {
  overflow: visible;
}
button,
select {
  text-transform: none;
}
button,
html input[type="button"],
input[type="reset"],
input[type="submit"] {
  -webkit-appearance: button;
  cursor: pointer;
}
button[disabled],
html input[disabled] {
  cursor: default;
}
button::-moz-focus-inner,
input::-moz-focus-inner {
  border: 0;
  padding: 0;
}
input {
  line-height: normal;
}
input[type="checkbox"],
input[type="radio"] {
  box-sizing: border-box;
  padding: 0;
}
input[type="number"]::-webkit-inner-spin-button,
input[type="number"]::-webkit-outer-spin-button {
  height: auto;
}
input[type="search"] {
  -webkit-appearance: textfield;
  box-sizing: content-box;
}
input[type="search"]::-webkit-search-cancel-button,
input[type="search"]::-webkit-search-decoration {
  -webkit-appearance: none;
}
fieldset {
  border: 1px solid #c0c0c0;
  margin: 0 2px;
  padding: 0.35em 0.625em 0.75em;
}
legend {
  border: 0;
  padding: 0;
}
textarea {
  overflow: auto;
}
optgroup {
  font-weight: bold;
}
table {
  border-collapse: collapse;
  border-spacing: 0;
}
td,
th {
  padding: 0;
}
/*! Source: https://github.com/h5bp/html5-boilerplate/blob/master/src/css/main.css */
@media print {
  *,
  *:before,
  *:after {
    background: transparent !important;
    box-shadow: none !important;
    text-shadow: none !important;
  }
  a,
  a:visited {
    text-decoration: underline;
  }
  a[href]:after {
    content: " (" attr(href) ")";
  }
  abbr[title]:after {
    content: " (" attr(title) ")";
  }
  a[href^="#"]:after,
  a[href^="javascript:"]:after {
    content: "";
  }
  pre,
  blockquote {
    border: 1px solid #999;
    page-break-inside: avoid;
  }
  thead {
    display: table-header-group;
  }
  tr,
  img {
    page-break-inside: avoid;
  }
  img {
    max-width: 100% !important;
  }
  p,
  h2,
  h3 {
    orphans: 3;
    widows: 3;
  }
  h2,
  h3 {
    page-break-after: avoid;
  }
  .navbar {
    display: none;
  }
  .btn > .caret,
  .dropup > .btn > .caret {
    border-top-color: #000 !important;
  }
  .label {
    border: 1px solid #000;
  }
  .table {
    border-collapse: collapse !important;
  }
  .table td,
  .table th {
    background-color: #fff !important;
  }
  .table-bordered th,
  .table-bordered td {
    border: 1px solid #ddd !important;
  }
}
@font-face {
  font-family: 'Glyphicons Halflings';
  src: url('../components/bootstrap/fonts/glyphicons-halflings-regular.eot');
  src: url('../components/bootstrap/fonts/glyphicons-halflings-regular.eot?#iefix') format('embedded-opentype'), url('../components/bootstrap/fonts/glyphicons-halflings-regular.woff2') format('woff2'), url('../components/bootstrap/fonts/glyphicons-halflings-regular.woff') format('woff'), url('../components/bootstrap/fonts/glyphicons-halflings-regular.ttf') format('truetype'), url('../components/bootstrap/fonts/glyphicons-halflings-regular.svg#glyphicons_halflingsregular') format('svg');
}
.glyphicon {
  position: relative;
  top: 1px;
  display: inline-block;
  font-family: 'Glyphicons Halflings';
  font-style: normal;
  font-weight: normal;
  line-height: 1;
  -webkit-font-smoothing: antialiased;
  -moz-osx-font-smoothing: grayscale;
}
.glyphicon-asterisk:before {
  content: "\002a";
}
.glyphicon-plus:before {
  content: "\002b";
}
.glyphicon-euro:before,
.glyphicon-eur:before {
  content: "\20ac";
}
.glyphicon-minus:before {
  content: "\2212";
}
.glyphicon-cloud:before {
  content: "\2601";
}
.glyphicon-envelope:before {
  content: "\2709";
}
.glyphicon-pencil:before {
  content: "\270f";
}
.glyphicon-glass:before {
  content: "\e001";
}
.glyphicon-music:before {
  content: "\e002";
}
.glyphicon-search:before {
  content: "\e003";
}
.glyphicon-heart:before {
  content: "\e005";
}
.glyphicon-star:before {
  content: "\e006";
}
.glyphicon-star-empty:before {
  content: "\e007";
}
.glyphicon-user:before {
  content: "\e008";
}
.glyphicon-film:before {
  content: "\e009";
}
.glyphicon-th-large:before {
  content: "\e010";
}
.glyphicon-th:before {
  content: "\e011";
}
.glyphicon-th-list:before {
  content: "\e012";
}
.glyphicon-ok:before {
  content: "\e013";
}
.glyphicon-remove:before {
  content: "\e014";
}
.glyphicon-zoom-in:before {
  content: "\e015";
}
.glyphicon-zoom-out:before {
  content: "\e016";
}
.glyphicon-off:before {
  content: "\e017";
}
.glyphicon-signal:before {
  content: "\e018";
}
.glyphicon-cog:before {
  content: "\e019";
}
.glyphicon-trash:before {
  content: "\e020";
}
.glyphicon-home:before {
  content: "\e021";
}
.glyphicon-file:before {
  content: "\e022";
}
.glyphicon-time:before {
  content: "\e023";
}
.glyphicon-road:before {
  content: "\e024";
}
.glyphicon-download-alt:before {
  content: "\e025";
}
.glyphicon-download:before {
  content: "\e026";
}
.glyphicon-upload:before {
  content: "\e027";
}
.glyphicon-inbox:before {
  content: "\e028";
}
.glyphicon-play-circle:before {
  content: "\e029";
}
.glyphicon-repeat:before {
  content: "\e030";
}
.glyphicon-refresh:before {
  content: "\e031";
}
.glyphicon-list-alt:before {
  content: "\e032";
}
.glyphicon-lock:before {
  content: "\e033";
}
.glyphicon-flag:before {
  content: "\e034";
}
.glyphicon-headphones:before {
  content: "\e035";
}
.glyphicon-volume-off:before {
  content: "\e036";
}
.glyphicon-volume-down:before {
  content: "\e037";
}
.glyphicon-volume-up:before {
  content: "\e038";
}
.glyphicon-qrcode:before {
  content: "\e039";
}
.glyphicon-barcode:before {
  content: "\e040";
}
.glyphicon-tag:before {
  content: "\e041";
}
.glyphicon-tags:before {
  content: "\e042";
}
.glyphicon-book:before {
  content: "\e043";
}
.glyphicon-bookmark:before {
  content: "\e044";
}
.glyphicon-print:before {
  content: "\e045";
}
.glyphicon-camera:before {
  content: "\e046";
}
.glyphicon-font:before {
  content: "\e047";
}
.glyphicon-bold:before {
  content: "\e048";
}
.glyphicon-italic:before {
  content: "\e049";
}
.glyphicon-text-height:before {
  content: "\e050";
}
.glyphicon-text-width:before {
  content: "\e051";
}
.glyphicon-align-left:before {
  content: "\e052";
}
.glyphicon-align-center:before {
  content: "\e053";
}
.glyphicon-align-right:before {
  content: "\e054";
}
.glyphicon-align-justify:before {
  content: "\e055";
}
.glyphicon-list:before {
  content: "\e056";
}
.glyphicon-indent-left:before {
  content: "\e057";
}
.glyphicon-indent-right:before {
  content: "\e058";
}
.glyphicon-facetime-video:before {
  content: "\e059";
}
.glyphicon-picture:before {
  content: "\e060";
}
.glyphicon-map-marker:before {
  content: "\e062";
}
.glyphicon-adjust:before {
  content: "\e063";
}
.glyphicon-tint:before {
  content: "\e064";
}
.glyphicon-edit:before {
  content: "\e065";
}
.glyphicon-share:before {
  content: "\e066";
}
.glyphicon-check:before {
  content: "\e067";
}
.glyphicon-move:before {
  content: "\e068";
}
.glyphicon-step-backward:before {
  content: "\e069";
}
.glyphicon-fast-backward:before {
  content: "\e070";
}
.glyphicon-backward:before {
  content: "\e071";
}
.glyphicon-play:before {
  content: "\e072";
}
.glyphicon-pause:before {
  content: "\e073";
}
.glyphicon-stop:before {
  content: "\e074";
}
.glyphicon-forward:before {
  content: "\e075";
}
.glyphicon-fast-forward:before {
  content: "\e076";
}
.glyphicon-step-forward:before {
  content: "\e077";
}
.glyphicon-eject:before {
  content: "\e078";
}
.glyphicon-chevron-left:before {
  content: "\e079";
}
.glyphicon-chevron-right:before {
  content: "\e080";
}
.glyphicon-plus-sign:before {
  content: "\e081";
}
.glyphicon-minus-sign:before {
  content: "\e082";
}
.glyphicon-remove-sign:before {
  content: "\e083";
}
.glyphicon-ok-sign:before {
  content: "\e084";
}
.glyphicon-question-sign:before {
  content: "\e085";
}
.glyphicon-info-sign:before {
  content: "\e086";
}
.glyphicon-screenshot:before {
  content: "\e087";
}
.glyphicon-remove-circle:before {
  content: "\e088";
}
.glyphicon-ok-circle:before {
  content: "\e089";
}
.glyphicon-ban-circle:before {
  content: "\e090";
}
.glyphicon-arrow-left:before {
  content: "\e091";
}
.glyphicon-arrow-right:before {
  content: "\e092";
}
.glyphicon-arrow-up:before {
  content: "\e093";
}
.glyphicon-arrow-down:before {
  content: "\e094";
}
.glyphicon-share-alt:before {
  content: "\e095";
}
.glyphicon-resize-full:before {
  content: "\e096";
}
.glyphicon-resize-small:before {
  content: "\e097";
}
.glyphicon-exclamation-sign:before {
  content: "\e101";
}
.glyphicon-gift:before {
  content: "\e102";
}
.glyphicon-leaf:before {
  content: "\e103";
}
.glyphicon-fire:before {
  content: "\e104";
}
.glyphicon-eye-open:before {
  content: "\e105";
}
.glyphicon-eye-close:before {
  content: "\e106";
}
.glyphicon-warning-sign:before {
  content: "\e107";
}
.glyphicon-plane:before {
  content: "\e108";
}
.glyphicon-calendar:before {
  content: "\e109";
}
.glyphicon-random:before {
  content: "\e110";
}
.glyphicon-comment:before {
  content: "\e111";
}
.glyphicon-magnet:before {
  content: "\e112";
}
.glyphicon-chevron-up:before {
  content: "\e113";
}
.glyphicon-chevron-down:before {
  content: "\e114";
}
.glyphicon-retweet:before {
  content: "\e115";
}
.glyphicon-shopping-cart:before {
  content: "\e116";
}
.glyphicon-folder-close:before {
  content: "\e117";
}
.glyphicon-folder-open:before {
  content: "\e118";
}
.glyphicon-resize-vertical:before {
  content: "\e119";
}
.glyphicon-resize-horizontal:before {
  content: "\e120";
}
.glyphicon-hdd:before {
  content: "\e121";
}
.glyphicon-bullhorn:before {
  content: "\e122";
}
.glyphicon-bell:before {
  content: "\e123";
}
.glyphicon-certificate:before {
  content: "\e124";
}
.glyphicon-thumbs-up:before {
  content: "\e125";
}
.glyphicon-thumbs-down:before {
  content: "\e126";
}
.glyphicon-hand-right:before {
  content: "\e127";
}
.glyphicon-hand-left:before {
  content: "\e128";
}
.glyphicon-hand-up:before {
  content: "\e129";
}
.glyphicon-hand-down:before {
  content: "\e130";
}
.glyphicon-circle-arrow-right:before {
  content: "\e131";
}
.glyphicon-circle-arrow-left:before {
  content: "\e132";
}
.glyphicon-circle-arrow-up:before {
  content: "\e133";
}
.glyphicon-circle-arrow-down:before {
  content: "\e134";
}
.glyphicon-globe:before {
  content: "\e135";
}
.glyphicon-wrench:before {
  content: "\e136";
}
.glyphicon-tasks:before {
  content: "\e137";
}
.glyphicon-filter:before {
  content: "\e138";
}
.glyphicon-briefcase:before {
  content: "\e139";
}
.glyphicon-fullscreen:before {
  content: "\e140";
}
.glyphicon-dashboard:before {
  content: "\e141";
}
.glyphicon-paperclip:before {
  content: "\e142";
}
.glyphicon-heart-empty:before {
  content: "\e143";
}
.glyphicon-link:before {
  content: "\e144";
}
.glyphicon-phone:before {
  content: "\e145";
}
.glyphicon-pushpin:before {
  content: "\e146";
}
.glyphicon-usd:before {
  content: "\e148";
}
.glyphicon-gbp:before {
  content: "\e149";
}
.glyphicon-sort:before {
  content: "\e150";
}
.glyphicon-sort-by-alphabet:before {
  content: "\e151";
}
.glyphicon-sort-by-alphabet-alt:before {
  content: "\e152";
}
.glyphicon-sort-by-order:before {
  content: "\e153";
}
.glyphicon-sort-by-order-alt:before {
  content: "\e154";
}
.glyphicon-sort-by-attributes:before {
  content: "\e155";
}
.glyphicon-sort-by-attributes-alt:before {
  content: "\e156";
}
.glyphicon-unchecked:before {
  content: "\e157";
}
.glyphicon-expand:before {
  content: "\e158";
}
.glyphicon-collapse-down:before {
  content: "\e159";
}
.glyphicon-collapse-up:before {
  content: "\e160";
}
.glyphicon-log-in:before {
  content: "\e161";
}
.glyphicon-flash:before {
  content: "\e162";
}
.glyphicon-log-out:before {
  content: "\e163";
}
.glyphicon-new-window:before {
  content: "\e164";
}
.glyphicon-record:before {
  content: "\e165";
}
.glyphicon-save:before {
  content: "\e166";
}
.glyphicon-open:before {
  content: "\e167";
}
.glyphicon-saved:before {
  content: "\e168";
}
.glyphicon-import:before {
  content: "\e169";
}
.glyphicon-export:before {
  content: "\e170";
}
.glyphicon-send:before {
  content: "\e171";
}
.glyphicon-floppy-disk:before {
  content: "\e172";
}
.glyphicon-floppy-saved:before {
  content: "\e173";
}
.glyphicon-floppy-remove:before {
  content: "\e174";
}
.glyphicon-floppy-save:before {
  content: "\e175";
}
.glyphicon-floppy-open:before {
  content: "\e176";
}
.glyphicon-credit-card:before {
  content: "\e177";
}
.glyphicon-transfer:before {
  content: "\e178";
}
.glyphicon-cutlery:before {
  content: "\e179";
}
.glyphicon-header:before {
  content: "\e180";
}
.glyphicon-compressed:before {
  content: "\e181";
}
.glyphicon-earphone:before {
  content: "\e182";
}
.glyphicon-phone-alt:before {
  content: "\e183";
}
.glyphicon-tower:before {
  content: "\e184";
}
.glyphicon-stats:before {
  content: "\e185";
}
.glyphicon-sd-video:before {
  content: "\e186";
}
.glyphicon-hd-video:before {
  content: "\e187";
}
.glyphicon-subtitles:before {
  content: "\e188";
}
.glyphicon-sound-stereo:before {
  content: "\e189";
}
.glyphicon-sound-dolby:before {
  content: "\e190";
}
.glyphicon-sound-5-1:before {
  content: "\e191";
}
.glyphicon-sound-6-1:before {
  content: "\e192";
}
.glyphicon-sound-7-1:before {
  content: "\e193";
}
.glyphicon-copyright-mark:before {
  content: "\e194";
}
.glyphicon-registration-mark:before {
  content: "\e195";
}
.glyphicon-cloud-download:before {
  content: "\e197";
}
.glyphicon-cloud-upload:before {
  content: "\e198";
}
.glyphicon-tree-conifer:before {
  content: "\e199";
}
.glyphicon-tree-deciduous:before {
  content: "\e200";
}
.glyphicon-cd:before {
  content: "\e201";
}
.glyphicon-save-file:before {
  content: "\e202";
}
.glyphicon-open-file:before {
  content: "\e203";
}
.glyphicon-level-up:before {
  content: "\e204";
}
.glyphicon-copy:before {
  content: "\e205";
}
.glyphicon-paste:before {
  content: "\e206";
}
.glyphicon-alert:before {
  content: "\e209";
}
.glyphicon-equalizer:before {
  content: "\e210";
}
.glyphicon-king:before {
  content: "\e211";
}
.glyphicon-queen:before {
  content: "\e212";
}
.glyphicon-pawn:before {
  content: "\e213";
}
.glyphicon-bishop:before {
  content: "\e214";
}
.glyphicon-knight:before {
  content: "\e215";
}
.glyphicon-baby-formula:before {
  content: "\e216";
}
.glyphicon-tent:before {
  content: "\26fa";
}
.glyphicon-blackboard:before {
  content: "\e218";
}
.glyphicon-bed:before {
  content: "\e219";
}
.glyphicon-apple:before {
  content: "\f8ff";
}
.glyphicon-erase:before {
  content: "\e221";
}
.glyphicon-hourglass:before {
  content: "\231b";
}
.glyphicon-lamp:before {
  content: "\e223";
}
.glyphicon-duplicate:before {
  content: "\e224";
}
.glyphicon-piggy-bank:before {
  content: "\e225";
}
.glyphicon-scissors:before {
  content: "\e226";
}
.glyphicon-bitcoin:before {
  content: "\e227";
}
.glyphicon-btc:before {
  content: "\e227";
}
.glyphicon-xbt:before {
  content: "\e227";
}
.glyphicon-yen:before {
  content: "\00a5";
}
.glyphicon-jpy:before {
  content: "\00a5";
}
.glyphicon-ruble:before {
  content: "\20bd";
}
.glyphicon-rub:before {
  content: "\20bd";
}
.glyphicon-scale:before {
  content: "\e230";
}
.glyphicon-ice-lolly:before {
  content: "\e231";
}
.glyphicon-ice-lolly-tasted:before {
  content: "\e232";
}
.glyphicon-education:before {
  content: "\e233";
}
.glyphicon-option-horizontal:before {
  content: "\e234";
}
.glyphicon-option-vertical:before {
  content: "\e235";
}
.glyphicon-menu-hamburger:before {
  content: "\e236";
}
.glyphicon-modal-window:before {
  content: "\e237";
}
.glyphicon-oil:before {
  content: "\e238";
}
.glyphicon-grain:before {
  content: "\e239";
}
.glyphicon-sunglasses:before {
  content: "\e240";
}
.glyphicon-text-size:before {
  content: "\e241";
}
.glyphicon-text-color:before {
  content: "\e242";
}
.glyphicon-text-background:before {
  content: "\e243";
}
.glyphicon-object-align-top:before {
  content: "\e244";
}
.glyphicon-object-align-bottom:before {
  content: "\e245";
}
.glyphicon-object-align-horizontal:before {
  content: "\e246";
}
.glyphicon-object-align-left:before {
  content: "\e247";
}
.glyphicon-object-align-vertical:before {
  content: "\e248";
}
.glyphicon-object-align-right:before {
  content: "\e249";
}
.glyphicon-triangle-right:before {
  content: "\e250";
}
.glyphicon-triangle-left:before {
  content: "\e251";
}
.glyphicon-triangle-bottom:before {
  content: "\e252";
}
.glyphicon-triangle-top:before {
  content: "\e253";
}
.glyphicon-console:before {
  content: "\e254";
}
.glyphicon-superscript:before {
  content: "\e255";
}
.glyphicon-subscript:before {
  content: "\e256";
}
.glyphicon-menu-left:before {
  content: "\e257";
}
.glyphicon-menu-right:before {
  content: "\e258";
}
.glyphicon-menu-down:before {
  content: "\e259";
}
.glyphicon-menu-up:before {
  content: "\e260";
}
* {
  -webkit-box-sizing: border-box;
  -moz-box-sizing: border-box;
  box-sizing: border-box;
}
*:before,
*:after {
  -webkit-box-sizing: border-box;
  -moz-box-sizing: border-box;
  box-sizing: border-box;
}
html {
  font-size: 10px;
  -webkit-tap-highlight-color: rgba(0, 0, 0, 0);
}
body {
  font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
  font-size: 13px;
  line-height: 1.42857143;
  color: #000;
  background-color: #fff;
}
input,
button,
select,
textarea {
  font-family: inherit;
  font-size: inherit;
  line-height: inherit;
}
a {
  color: #337ab7;
  text-decoration: none;
}
a:hover,
a:focus {
  color: #23527c;
  text-decoration: underline;
}
a:focus {
  outline: 5px auto -webkit-focus-ring-color;
  outline-offset: -2px;
}
figure {
  margin: 0;
}
img {
  vertical-align: middle;
}
.img-responsive,
.thumbnail > img,
.thumbnail a > img,
.carousel-inner > .item > img,
.carousel-inner > .item > a > img {
  display: block;
  max-width: 100%;
  height: auto;
}
.img-rounded {
  border-radius: 3px;
}
.img-thumbnail {
  padding: 4px;
  line-height: 1.42857143;
  background-color: #fff;
  border: 1px solid #ddd;
  border-radius: 2px;
  -webkit-transition: all 0.2s ease-in-out;
  -o-transition: all 0.2s ease-in-out;
  transition: all 0.2s ease-in-out;
  display: inline-block;
  max-width: 100%;
  height: auto;
}
.img-circle {
  border-radius: 50%;
}
hr {
  margin-top: 18px;
  margin-bottom: 18px;
  border: 0;
  border-top: 1px solid #eeeeee;
}
.sr-only {
  position: absolute;
  width: 1px;
  height: 1px;
  margin: -1px;
  padding: 0;
  overflow: hidden;
  clip: rect(0, 0, 0, 0);
  border: 0;
}
.sr-only-focusable:active,
.sr-only-focusable:focus {
  position: static;
  width: auto;
  height: auto;
  margin: 0;
  overflow: visible;
  clip: auto;
}
[role="button"] {
  cursor: pointer;
}
h1,
h2,
h3,
h4,
h5,
h6,
.h1,
.h2,
.h3,
.h4,
.h5,
.h6 {
  font-family: inherit;
  font-weight: 500;
  line-height: 1.1;
  color: inherit;
}
h1 small,
h2 small,
h3 small,
h4 small,
h5 small,
h6 small,
.h1 small,
.h2 small,
.h3 small,
.h4 small,
.h5 small,
.h6 small,
h1 .small,
h2 .small,
h3 .small,
h4 .small,
h5 .small,
h6 .small,
.h1 .small,
.h2 .small,
.h3 .small,
.h4 .small,
.h5 .small,
.h6 .small {
  font-weight: normal;
  line-height: 1;
  color: #777777;
}
h1,
.h1,
h2,
.h2,
h3,
.h3 {
  margin-top: 18px;
  margin-bottom: 9px;
}
h1 small,
.h1 small,
h2 small,
.h2 small,
h3 small,
.h3 small,
h1 .small,
.h1 .small,
h2 .small,
.h2 .small,
h3 .small,
.h3 .small {
  font-size: 65%;
}
h4,
.h4,
h5,
.h5,
h6,
.h6 {
  margin-top: 9px;
  margin-bottom: 9px;
}
h4 small,
.h4 small,
h5 small,
.h5 small,
h6 small,
.h6 small,
h4 .small,
.h4 .small,
h5 .small,
.h5 .small,
h6 .small,
.h6 .small {
  font-size: 75%;
}
h1,
.h1 {
  font-size: 33px;
}
h2,
.h2 {
  font-size: 27px;
}
h3,
.h3 {
  font-size: 23px;
}
h4,
.h4 {
  font-size: 17px;
}
h5,
.h5 {
  font-size: 13px;
}
h6,
.h6 {
  font-size: 12px;
}
p {
  margin: 0 0 9px;
}
.lead {
  margin-bottom: 18px;
  font-size: 14px;
  font-weight: 300;
  line-height: 1.4;
}
@media (min-width: 768px) {
  .lead {
    font-size: 19.5px;
  }
}
small,
.small {
  font-size: 92%;
}
mark,
.mark {
  background-color: #fcf8e3;
  padding: .2em;
}
.text-left {
  text-align: left;
}
.text-right {
  text-align: right;
}
.text-center {
  text-align: center;
}
.text-justify {
  text-align: justify;
}
.text-nowrap {
  white-space: nowrap;
}
.text-lowercase {
  text-transform: lowercase;
}
.text-uppercase {
  text-transform: uppercase;
}
.text-capitalize {
  text-transform: capitalize;
}
.text-muted {
  color: #777777;
}
.text-primary {
  color: #337ab7;
}
a.text-primary:hover,
a.text-primary:focus {
  color: #286090;
}
.text-success {
  color: #3c763d;
}
a.text-success:hover,
a.text-success:focus {
  color: #2b542c;
}
.text-info {
  color: #31708f;
}
a.text-info:hover,
a.text-info:focus {
  color: #245269;
}
.text-warning {
  color: #8a6d3b;
}
a.text-warning:hover,
a.text-warning:focus {
  color: #66512c;
}
.text-danger {
  color: #a94442;
}
a.text-danger:hover,
a.text-danger:focus {
  color: #843534;
}
.bg-primary {
  color: #fff;
  background-color: #337ab7;
}
a.bg-primary:hover,
a.bg-primary:focus {
  background-color: #286090;
}
.bg-success {
  background-color: #dff0d8;
}
a.bg-success:hover,
a.bg-success:focus {
  background-color: #c1e2b3;
}
.bg-info {
  background-color: #d9edf7;
}
a.bg-info:hover,
a.bg-info:focus {
  background-color: #afd9ee;
}
.bg-warning {
  background-color: #fcf8e3;
}
a.bg-warning:hover,
a.bg-warning:focus {
  background-color: #f7ecb5;
}
.bg-danger {
  background-color: #f2dede;
}
a.bg-danger:hover,
a.bg-danger:focus {
  background-color: #e4b9b9;
}
.page-header {
  padding-bottom: 8px;
  margin: 36px 0 18px;
  border-bottom: 1px solid #eeeeee;
}
ul,
ol {
  margin-top: 0;
  margin-bottom: 9px;
}
ul ul,
ol ul,
ul ol,
ol ol {
  margin-bottom: 0;
}
.list-unstyled {
  padding-left: 0;
  list-style: none;
}
.list-inline {
  padding-left: 0;
  list-style: none;
  margin-left: -5px;
}
.list-inline > li {
  display: inline-block;
  padding-left: 5px;
  padding-right: 5px;
}
dl {
  margin-top: 0;
  margin-bottom: 18px;
}
dt,
dd {
  line-height: 1.42857143;
}
dt {
  font-weight: bold;
}
dd {
  margin-left: 0;
}
@media (min-width: 541px) {
  .dl-horizontal dt {
    float: left;
    width: 160px;
    clear: left;
    text-align: right;
    overflow: hidden;
    text-overflow: ellipsis;
    white-space: nowrap;
  }
  .dl-horizontal dd {
    margin-left: 180px;
  }
}
abbr[title],
abbr[data-original-title] {
  cursor: help;
  border-bottom: 1px dotted #777777;
}
.initialism {
  font-size: 90%;
  text-transform: uppercase;
}
blockquote {
  padding: 9px 18px;
  margin: 0 0 18px;
  font-size: inherit;
  border-left: 5px solid #eeeeee;
}
blockquote p:last-child,
blockquote ul:last-child,
blockquote ol:last-child {
  margin-bottom: 0;
}
blockquote footer,
blockquote small,
blockquote .small {
  display: block;
  font-size: 80%;
  line-height: 1.42857143;
  color: #777777;
}
blockquote footer:before,
blockquote small:before,
blockquote .small:before {
  content: '\2014 \00A0';
}
.blockquote-reverse,
blockquote.pull-right {
  padding-right: 15px;
  padding-left: 0;
  border-right: 5px solid #eeeeee;
  border-left: 0;
  text-align: right;
}
.blockquote-reverse footer:before,
blockquote.pull-right footer:before,
.blockquote-reverse small:before,
blockquote.pull-right small:before,
.blockquote-reverse .small:before,
blockquote.pull-right .small:before {
  content: '';
}
.blockquote-reverse footer:after,
blockquote.pull-right footer:after,
.blockquote-reverse small:after,
blockquote.pull-right small:after,
.blockquote-reverse .small:after,
blockquote.pull-right .small:after {
  content: '\00A0 \2014';
}
address {
  margin-bottom: 18px;
  font-style: normal;
  line-height: 1.42857143;
}
code,
kbd,
pre,
samp {
  font-family: monospace;
}
code {
  padding: 2px 4px;
  font-size: 90%;
  color: #c7254e;
  background-color: #f9f2f4;
  border-radius: 2px;
}
kbd {
  padding: 2px 4px;
  font-size: 90%;
  color: #888;
  background-color: transparent;
  border-radius: 1px;
  box-shadow: inset 0 -1px 0 rgba(0, 0, 0, 0.25);
}
kbd kbd {
  padding: 0;
  font-size: 100%;
  font-weight: bold;
  box-shadow: none;
}
pre {
  display: block;
  padding: 8.5px;
  margin: 0 0 9px;
  font-size: 12px;
  line-height: 1.42857143;
  word-break: break-all;
  word-wrap: break-word;
  color: #333333;
  background-color: #f5f5f5;
  border: 1px solid #ccc;
  border-radius: 2px;
}
pre code {
  padding: 0;
  font-size: inherit;
  color: inherit;
  white-space: pre-wrap;
  background-color: transparent;
  border-radius: 0;
}
.pre-scrollable {
  max-height: 340px;
  overflow-y: scroll;
}
.container {
  margin-right: auto;
  margin-left: auto;
  padding-left: 0px;
  padding-right: 0px;
}
@media (min-width: 768px) {
  .container {
    width: 768px;
  }
}
@media (min-width: 992px) {
  .container {
    width: 940px;
  }
}
@media (min-width: 1200px) {
  .container {
    width: 1140px;
  }
}
.container-fluid {
  margin-right: auto;
  margin-left: auto;
  padding-left: 0px;
  padding-right: 0px;
}
.row {
  margin-left: 0px;
  margin-right: 0px;
}
.col-xs-1, .col-sm-1, .col-md-1, .col-lg-1, .col-xs-2, .col-sm-2, .col-md-2, .col-lg-2, .col-xs-3, .col-sm-3, .col-md-3, .col-lg-3, .col-xs-4, .col-sm-4, .col-md-4, .col-lg-4, .col-xs-5, .col-sm-5, .col-md-5, .col-lg-5, .col-xs-6, .col-sm-6, .col-md-6, .col-lg-6, .col-xs-7, .col-sm-7, .col-md-7, .col-lg-7, .col-xs-8, .col-sm-8, .col-md-8, .col-lg-8, .col-xs-9, .col-sm-9, .col-md-9, .col-lg-9, .col-xs-10, .col-sm-10, .col-md-10, .col-lg-10, .col-xs-11, .col-sm-11, .col-md-11, .col-lg-11, .col-xs-12, .col-sm-12, .col-md-12, .col-lg-12 {
  position: relative;
  min-height: 1px;
  padding-left: 0px;
  padding-right: 0px;
}
.col-xs-1, .col-xs-2, .col-xs-3, .col-xs-4, .col-xs-5, .col-xs-6, .col-xs-7, .col-xs-8, .col-xs-9, .col-xs-10, .col-xs-11, .col-xs-12 {
  float: left;
}
.col-xs-12 {
  width: 100%;
}
.col-xs-11 {
  width: 91.66666667%;
}
.col-xs-10 {
  width: 83.33333333%;
}
.col-xs-9 {
  width: 75%;
}
.col-xs-8 {
  width: 66.66666667%;
}
.col-xs-7 {
  width: 58.33333333%;
}
.col-xs-6 {
  width: 50%;
}
.col-xs-5 {
  width: 41.66666667%;
}
.col-xs-4 {
  width: 33.33333333%;
}
.col-xs-3 {
  width: 25%;
}
.col-xs-2 {
  width: 16.66666667%;
}
.col-xs-1 {
  width: 8.33333333%;
}
.col-xs-pull-12 {
  right: 100%;
}
.col-xs-pull-11 {
  right: 91.66666667%;
}
.col-xs-pull-10 {
  right: 83.33333333%;
}
.col-xs-pull-9 {
  right: 75%;
}
.col-xs-pull-8 {
  right: 66.66666667%;
}
.col-xs-pull-7 {
  right: 58.33333333%;
}
.col-xs-pull-6 {
  right: 50%;
}
.col-xs-pull-5 {
  right: 41.66666667%;
}
.col-xs-pull-4 {
  right: 33.33333333%;
}
.col-xs-pull-3 {
  right: 25%;
}
.col-xs-pull-2 {
  right: 16.66666667%;
}
.col-xs-pull-1 {
  right: 8.33333333%;
}
.col-xs-pull-0 {
  right: auto;
}
.col-xs-push-12 {
  left: 100%;
}
.col-xs-push-11 {
  left: 91.66666667%;
}
.col-xs-push-10 {
  left: 83.33333333%;
}
.col-xs-push-9 {
  left: 75%;
}
.col-xs-push-8 {
  left: 66.66666667%;
}
.col-xs-push-7 {
  left: 58.33333333%;
}
.col-xs-push-6 {
  left: 50%;
}
.col-xs-push-5 {
  left: 41.66666667%;
}
.col-xs-push-4 {
  left: 33.33333333%;
}
.col-xs-push-3 {
  left: 25%;
}
.col-xs-push-2 {
  left: 16.66666667%;
}
.col-xs-push-1 {
  left: 8.33333333%;
}
.col-xs-push-0 {
  left: auto;
}
.col-xs-offset-12 {
  margin-left: 100%;
}
.col-xs-offset-11 {
  margin-left: 91.66666667%;
}
.col-xs-offset-10 {
  margin-left: 83.33333333%;
}
.col-xs-offset-9 {
  margin-left: 75%;
}
.col-xs-offset-8 {
  margin-left: 66.66666667%;
}
.col-xs-offset-7 {
  margin-left: 58.33333333%;
}
.col-xs-offset-6 {
  margin-left: 50%;
}
.col-xs-offset-5 {
  margin-left: 41.66666667%;
}
.col-xs-offset-4 {
  margin-left: 33.33333333%;
}
.col-xs-offset-3 {
  margin-left: 25%;
}
.col-xs-offset-2 {
  margin-left: 16.66666667%;
}
.col-xs-offset-1 {
  margin-left: 8.33333333%;
}
.col-xs-offset-0 {
  margin-left: 0%;
}
@media (min-width: 768px) {
  .col-sm-1, .col-sm-2, .col-sm-3, .col-sm-4, .col-sm-5, .col-sm-6, .col-sm-7, .col-sm-8, .col-sm-9, .col-sm-10, .col-sm-11, .col-sm-12 {
    float: left;
  }
  .col-sm-12 {
    width: 100%;
  }
  .col-sm-11 {
    width: 91.66666667%;
  }
  .col-sm-10 {
    width: 83.33333333%;
  }
  .col-sm-9 {
    width: 75%;
  }
  .col-sm-8 {
    width: 66.66666667%;
  }
  .col-sm-7 {
    width: 58.33333333%;
  }
  .col-sm-6 {
    width: 50%;
  }
  .col-sm-5 {
    width: 41.66666667%;
  }
  .col-sm-4 {
    width: 33.33333333%;
  }
  .col-sm-3 {
    width: 25%;
  }
  .col-sm-2 {
    width: 16.66666667%;
  }
  .col-sm-1 {
    width: 8.33333333%;
  }
  .col-sm-pull-12 {
    right: 100%;
  }
  .col-sm-pull-11 {
    right: 91.66666667%;
  }
  .col-sm-pull-10 {
    right: 83.33333333%;
  }
  .col-sm-pull-9 {
    right: 75%;
  }
  .col-sm-pull-8 {
    right: 66.66666667%;
  }
  .col-sm-pull-7 {
    right: 58.33333333%;
  }
  .col-sm-pull-6 {
    right: 50%;
  }
  .col-sm-pull-5 {
    right: 41.66666667%;
  }
  .col-sm-pull-4 {
    right: 33.33333333%;
  }
  .col-sm-pull-3 {
    right: 25%;
  }
  .col-sm-pull-2 {
    right: 16.66666667%;
  }
  .col-sm-pull-1 {
    right: 8.33333333%;
  }
  .col-sm-pull-0 {
    right: auto;
  }
  .col-sm-push-12 {
    left: 100%;
  }
  .col-sm-push-11 {
    left: 91.66666667%;
  }
  .col-sm-push-10 {
    left: 83.33333333%;
  }
  .col-sm-push-9 {
    left: 75%;
  }
  .col-sm-push-8 {
    left: 66.66666667%;
  }
  .col-sm-push-7 {
    left: 58.33333333%;
  }
  .col-sm-push-6 {
    left: 50%;
  }
  .col-sm-push-5 {
    left: 41.66666667%;
  }
  .col-sm-push-4 {
    left: 33.33333333%;
  }
  .col-sm-push-3 {
    left: 25%;
  }
  .col-sm-push-2 {
    left: 16.66666667%;
  }
  .col-sm-push-1 {
    left: 8.33333333%;
  }
  .col-sm-push-0 {
    left: auto;
  }
  .col-sm-offset-12 {
    margin-left: 100%;
  }
  .col-sm-offset-11 {
    margin-left: 91.66666667%;
  }
  .col-sm-offset-10 {
    margin-left: 83.33333333%;
  }
  .col-sm-offset-9 {
    margin-left: 75%;
  }
  .col-sm-offset-8 {
    margin-left: 66.66666667%;
  }
  .col-sm-offset-7 {
    margin-left: 58.33333333%;
  }
  .col-sm-offset-6 {
    margin-left: 50%;
  }
  .col-sm-offset-5 {
    margin-left: 41.66666667%;
  }
  .col-sm-offset-4 {
    margin-left: 33.33333333%;
  }
  .col-sm-offset-3 {
    margin-left: 25%;
  }
  .col-sm-offset-2 {
    margin-left: 16.66666667%;
  }
  .col-sm-offset-1 {
    margin-left: 8.33333333%;
  }
  .col-sm-offset-0 {
    margin-left: 0%;
  }
}
@media (min-width: 992px) {
  .col-md-1, .col-md-2, .col-md-3, .col-md-4, .col-md-5, .col-md-6, .col-md-7, .col-md-8, .col-md-9, .col-md-10, .col-md-11, .col-md-12 {
    float: left;
  }
  .col-md-12 {
    width: 100%;
  }
  .col-md-11 {
    width: 91.66666667%;
  }
  .col-md-10 {
    width: 83.33333333%;
  }
  .col-md-9 {
    width: 75%;
  }
  .col-md-8 {
    width: 66.66666667%;
  }
  .col-md-7 {
    width: 58.33333333%;
  }
  .col-md-6 {
    width: 50%;
  }
  .col-md-5 {
    width: 41.66666667%;
  }
  .col-md-4 {
    width: 33.33333333%;
  }
  .col-md-3 {
    width: 25%;
  }
  .col-md-2 {
    width: 16.66666667%;
  }
  .col-md-1 {
    width: 8.33333333%;
  }
  .col-md-pull-12 {
    right: 100%;
  }
  .col-md-pull-11 {
    right: 91.66666667%;
  }
  .col-md-pull-10 {
    right: 83.33333333%;
  }
  .col-md-pull-9 {
    right: 75%;
  }
  .col-md-pull-8 {
    right: 66.66666667%;
  }
  .col-md-pull-7 {
    right: 58.33333333%;
  }
  .col-md-pull-6 {
    right: 50%;
  }
  .col-md-pull-5 {
    right: 41.66666667%;
  }
  .col-md-pull-4 {
    right: 33.33333333%;
  }
  .col-md-pull-3 {
    right: 25%;
  }
  .col-md-pull-2 {
    right: 16.66666667%;
  }
  .col-md-pull-1 {
    right: 8.33333333%;
  }
  .col-md-pull-0 {
    right: auto;
  }
  .col-md-push-12 {
    left: 100%;
  }
  .col-md-push-11 {
    left: 91.66666667%;
  }
  .col-md-push-10 {
    left: 83.33333333%;
  }
  .col-md-push-9 {
    left: 75%;
  }
  .col-md-push-8 {
    left: 66.66666667%;
  }
  .col-md-push-7 {
    left: 58.33333333%;
  }
  .col-md-push-6 {
    left: 50%;
  }
  .col-md-push-5 {
    left: 41.66666667%;
  }
  .col-md-push-4 {
    left: 33.33333333%;
  }
  .col-md-push-3 {
    left: 25%;
  }
  .col-md-push-2 {
    left: 16.66666667%;
  }
  .col-md-push-1 {
    left: 8.33333333%;
  }
  .col-md-push-0 {
    left: auto;
  }
  .col-md-offset-12 {
    margin-left: 100%;
  }
  .col-md-offset-11 {
    margin-left: 91.66666667%;
  }
  .col-md-offset-10 {
    margin-left: 83.33333333%;
  }
  .col-md-offset-9 {
    margin-left: 75%;
  }
  .col-md-offset-8 {
    margin-left: 66.66666667%;
  }
  .col-md-offset-7 {
    margin-left: 58.33333333%;
  }
  .col-md-offset-6 {
    margin-left: 50%;
  }
  .col-md-offset-5 {
    margin-left: 41.66666667%;
  }
  .col-md-offset-4 {
    margin-left: 33.33333333%;
  }
  .col-md-offset-3 {
    margin-left: 25%;
  }
  .col-md-offset-2 {
    margin-left: 16.66666667%;
  }
  .col-md-offset-1 {
    margin-left: 8.33333333%;
  }
  .col-md-offset-0 {
    margin-left: 0%;
  }
}
@media (min-width: 1200px) {
  .col-lg-1, .col-lg-2, .col-lg-3, .col-lg-4, .col-lg-5, .col-lg-6, .col-lg-7, .col-lg-8, .col-lg-9, .col-lg-10, .col-lg-11, .col-lg-12 {
    float: left;
  }
  .col-lg-12 {
    width: 100%;
  }
  .col-lg-11 {
    width: 91.66666667%;
  }
  .col-lg-10 {
    width: 83.33333333%;
  }
  .col-lg-9 {
    width: 75%;
  }
  .col-lg-8 {
    width: 66.66666667%;
  }
  .col-lg-7 {
    width: 58.33333333%;
  }
  .col-lg-6 {
    width: 50%;
  }
  .col-lg-5 {
    width: 41.66666667%;
  }
  .col-lg-4 {
    width: 33.33333333%;
  }
  .col-lg-3 {
    width: 25%;
  }
  .col-lg-2 {
    width: 16.66666667%;
  }
  .col-lg-1 {
    width: 8.33333333%;
  }
  .col-lg-pull-12 {
    right: 100%;
  }
  .col-lg-pull-11 {
    right: 91.66666667%;
  }
  .col-lg-pull-10 {
    right: 83.33333333%;
  }
  .col-lg-pull-9 {
    right: 75%;
  }
  .col-lg-pull-8 {
    right: 66.66666667%;
  }
  .col-lg-pull-7 {
    right: 58.33333333%;
  }
  .col-lg-pull-6 {
    right: 50%;
  }
  .col-lg-pull-5 {
    right: 41.66666667%;
  }
  .col-lg-pull-4 {
    right: 33.33333333%;
  }
  .col-lg-pull-3 {
    right: 25%;
  }
  .col-lg-pull-2 {
    right: 16.66666667%;
  }
  .col-lg-pull-1 {
    right: 8.33333333%;
  }
  .col-lg-pull-0 {
    right: auto;
  }
  .col-lg-push-12 {
    left: 100%;
  }
  .col-lg-push-11 {
    left: 91.66666667%;
  }
  .col-lg-push-10 {
    left: 83.33333333%;
  }
  .col-lg-push-9 {
    left: 75%;
  }
  .col-lg-push-8 {
    left: 66.66666667%;
  }
  .col-lg-push-7 {
    left: 58.33333333%;
  }
  .col-lg-push-6 {
    left: 50%;
  }
  .col-lg-push-5 {
    left: 41.66666667%;
  }
  .col-lg-push-4 {
    left: 33.33333333%;
  }
  .col-lg-push-3 {
    left: 25%;
  }
  .col-lg-push-2 {
    left: 16.66666667%;
  }
  .col-lg-push-1 {
    left: 8.33333333%;
  }
  .col-lg-push-0 {
    left: auto;
  }
  .col-lg-offset-12 {
    margin-left: 100%;
  }
  .col-lg-offset-11 {
    margin-left: 91.66666667%;
  }
  .col-lg-offset-10 {
    margin-left: 83.33333333%;
  }
  .col-lg-offset-9 {
    margin-left: 75%;
  }
  .col-lg-offset-8 {
    margin-left: 66.66666667%;
  }
  .col-lg-offset-7 {
    margin-left: 58.33333333%;
  }
  .col-lg-offset-6 {
    margin-left: 50%;
  }
  .col-lg-offset-5 {
    margin-left: 41.66666667%;
  }
  .col-lg-offset-4 {
    margin-left: 33.33333333%;
  }
  .col-lg-offset-3 {
    margin-left: 25%;
  }
  .col-lg-offset-2 {
    margin-left: 16.66666667%;
  }
  .col-lg-offset-1 {
    margin-left: 8.33333333%;
  }
  .col-lg-offset-0 {
    margin-left: 0%;
  }
}
table {
  background-color: transparent;
}
caption {
  padding-top: 8px;
  padding-bottom: 8px;
  color: #777777;
  text-align: left;
}
th {
  text-align: left;
}
.table {
  width: 100%;
  max-width: 100%;
  margin-bottom: 18px;
}
.table > thead > tr > th,
.table > tbody > tr > th,
.table > tfoot > tr > th,
.table > thead > tr > td,
.table > tbody > tr > td,
.table > tfoot > tr > td {
  padding: 8px;
  line-height: 1.42857143;
  vertical-align: top;
  border-top: 1px solid #ddd;
}
.table > thead > tr > th {
  vertical-align: bottom;
  border-bottom: 2px solid #ddd;
}
.table > caption + thead > tr:first-child > th,
.table > colgroup + thead > tr:first-child > th,
.table > thead:first-child > tr:first-child > th,
.table > caption + thead > tr:first-child > td,
.table > colgroup + thead > tr:first-child > td,
.table > thead:first-child > tr:first-child > td {
  border-top: 0;
}
.table > tbody + tbody {
  border-top: 2px solid #ddd;
}
.table .table {
  background-color: #fff;
}
.table-condensed > thead > tr > th,
.table-condensed > tbody > tr > th,
.table-condensed > tfoot > tr > th,
.table-condensed > thead > tr > td,
.table-condensed > tbody > tr > td,
.table-condensed > tfoot > tr > td {
  padding: 5px;
}
.table-bordered {
  border: 1px solid #ddd;
}
.table-bordered > thead > tr > th,
.table-bordered > tbody > tr > th,
.table-bordered > tfoot > tr > th,
.table-bordered > thead > tr > td,
.table-bordered > tbody > tr > td,
.table-bordered > tfoot > tr > td {
  border: 1px solid #ddd;
}
.table-bordered > thead > tr > th,
.table-bordered > thead > tr > td {
  border-bottom-width: 2px;
}
.table-striped > tbody > tr:nth-of-type(odd) {
  background-color: #f9f9f9;
}
.table-hover > tbody > tr:hover {
  background-color: #f5f5f5;
}
table col[class*="col-"] {
  position: static;
  float: none;
  display: table-column;
}
table td[class*="col-"],
table th[class*="col-"] {
  position: static;
  float: none;
  display: table-cell;
}
.table > thead > tr > td.active,
.table > tbody > tr > td.active,
.table > tfoot > tr > td.active,
.table > thead > tr > th.active,
.table > tbody > tr > th.active,
.table > tfoot > tr > th.active,
.table > thead > tr.active > td,
.table > tbody > tr.active > td,
.table > tfoot > tr.active > td,
.table > thead > tr.active > th,
.table > tbody > tr.active > th,
.table > tfoot > tr.active > th {
  background-color: #f5f5f5;
}
.table-hover > tbody > tr > td.active:hover,
.table-hover > tbody > tr > th.active:hover,
.table-hover > tbody > tr.active:hover > td,
.table-hover > tbody > tr:hover > .active,
.table-hover > tbody > tr.active:hover > th {
  background-color: #e8e8e8;
}
.table > thead > tr > td.success,
.table > tbody > tr > td.success,
.table > tfoot > tr > td.success,
.table > thead > tr > th.success,
.table > tbody > tr > th.success,
.table > tfoot > tr > th.success,
.table > thead > tr.success > td,
.table > tbody > tr.success > td,
.table > tfoot > tr.success > td,
.table > thead > tr.success > th,
.table > tbody > tr.success > th,
.table > tfoot > tr.success > th {
  background-color: #dff0d8;
}
.table-hover > tbody > tr > td.success:hover,
.table-hover > tbody > tr > th.success:hover,
.table-hover > tbody > tr.success:hover > td,
.table-hover > tbody > tr:hover > .success,
.table-hover > tbody > tr.success:hover > th {
  background-color: #d0e9c6;
}
.table > thead > tr > td.info,
.table > tbody > tr > td.info,
.table > tfoot > tr > td.info,
.table > thead > tr > th.info,
.table > tbody > tr > th.info,
.table > tfoot > tr > th.info,
.table > thead > tr.info > td,
.table > tbody > tr.info > td,
.table > tfoot > tr.info > td,
.table > thead > tr.info > th,
.table > tbody > tr.info > th,
.table > tfoot > tr.info > th {
  background-color: #d9edf7;
}
.table-hover > tbody > tr > td.info:hover,
.table-hover > tbody > tr > th.info:hover,
.table-hover > tbody > tr.info:hover > td,
.table-hover > tbody > tr:hover > .info,
.table-hover > tbody > tr.info:hover > th {
  background-color: #c4e3f3;
}
.table > thead > tr > td.warning,
.table > tbody > tr > td.warning,
.table > tfoot > tr > td.warning,
.table > thead > tr > th.warning,
.table > tbody > tr > th.warning,
.table > tfoot > tr > th.warning,
.table > thead > tr.warning > td,
.table > tbody > tr.warning > td,
.table > tfoot > tr.warning > td,
.table > thead > tr.warning > th,
.table > tbody > tr.warning > th,
.table > tfoot > tr.warning > th {
  background-color: #fcf8e3;
}
.table-hover > tbody > tr > td.warning:hover,
.table-hover > tbody > tr > th.warning:hover,
.table-hover > tbody > tr.warning:hover > td,
.table-hover > tbody > tr:hover > .warning,
.table-hover > tbody > tr.warning:hover > th {
  background-color: #faf2cc;
}
.table > thead > tr > td.danger,
.table > tbody > tr > td.danger,
.table > tfoot > tr > td.danger,
.table > thead > tr > th.danger,
.table > tbody > tr > th.danger,
.table > tfoot > tr > th.danger,
.table > thead > tr.danger > td,
.table > tbody > tr.danger > td,
.table > tfoot > tr.danger > td,
.table > thead > tr.danger > th,
.table > tbody > tr.danger > th,
.table > tfoot > tr.danger > th {
  background-color: #f2dede;
}
.table-hover > tbody > tr > td.danger:hover,
.table-hover > tbody > tr > th.danger:hover,
.table-hover > tbody > tr.danger:hover > td,
.table-hover > tbody > tr:hover > .danger,
.table-hover > tbody > tr.danger:hover > th {
  background-color: #ebcccc;
}
.table-responsive {
  overflow-x: auto;
  min-height: 0.01%;
}
@media screen and (max-width: 767px) {
  .table-responsive {
    width: 100%;
    margin-bottom: 13.5px;
    overflow-y: hidden;
    -ms-overflow-style: -ms-autohiding-scrollbar;
    border: 1px solid #ddd;
  }
  .table-responsive > .table {
    margin-bottom: 0;
  }
  .table-responsive > .table > thead > tr > th,
  .table-responsive > .table > tbody > tr > th,
  .table-responsive > .table > tfoot > tr > th,
  .table-responsive > .table > thead > tr > td,
  .table-responsive > .table > tbody > tr > td,
  .table-responsive > .table > tfoot > tr > td {
    white-space: nowrap;
  }
  .table-responsive > .table-bordered {
    border: 0;
  }
  .table-responsive > .table-bordered > thead > tr > th:first-child,
  .table-responsive > .table-bordered > tbody > tr > th:first-child,
  .table-responsive > .table-bordered > tfoot > tr > th:first-child,
  .table-responsive > .table-bordered > thead > tr > td:first-child,
  .table-responsive > .table-bordered > tbody > tr > td:first-child,
  .table-responsive > .table-bordered > tfoot > tr > td:first-child {
    border-left: 0;
  }
  .table-responsive > .table-bordered > thead > tr > th:last-child,
  .table-responsive > .table-bordered > tbody > tr > th:last-child,
  .table-responsive > .table-bordered > tfoot > tr > th:last-child,
  .table-responsive > .table-bordered > thead > tr > td:last-child,
  .table-responsive > .table-bordered > tbody > tr > td:last-child,
  .table-responsive > .table-bordered > tfoot > tr > td:last-child {
    border-right: 0;
  }
  .table-responsive > .table-bordered > tbody > tr:last-child > th,
  .table-responsive > .table-bordered > tfoot > tr:last-child > th,
  .table-responsive > .table-bordered > tbody > tr:last-child > td,
  .table-responsive > .table-bordered > tfoot > tr:last-child > td {
    border-bottom: 0;
  }
}
fieldset {
  padding: 0;
  margin: 0;
  border: 0;
  min-width: 0;
}
legend {
  display: block;
  width: 100%;
  padding: 0;
  margin-bottom: 18px;
  font-size: 19.5px;
  line-height: inherit;
  color: #333333;
  border: 0;
  border-bottom: 1px solid #e5e5e5;
}
label {
  display: inline-block;
  max-width: 100%;
  margin-bottom: 5px;
  font-weight: bold;
}
input[type="search"] {
  -webkit-box-sizing: border-box;
  -moz-box-sizing: border-box;
  box-sizing: border-box;
}
input[type="radio"],
input[type="checkbox"] {
  margin: 4px 0 0;
  margin-top: 1px \9;
  line-height: normal;
}
input[type="file"] {
  display: block;
}
input[type="range"] {
  display: block;
  width: 100%;
}
select[multiple],
select[size] {
  height: auto;
}
input[type="file"]:focus,
input[type="radio"]:focus,
input[type="checkbox"]:focus {
  outline: 5px auto -webkit-focus-ring-color;
  outline-offset: -2px;
}
output {
  display: block;
  padding-top: 7px;
  font-size: 13px;
  line-height: 1.42857143;
  color: #555555;
}
.form-control {
  display: block;
  width: 100%;
  height: 32px;
  padding: 6px 12px;
  font-size: 13px;
  line-height: 1.42857143;
  color: #555555;
  background-color: #fff;
  background-image: none;
  border: 1px solid #ccc;
  border-radius: 2px;
  -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
  box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
  -webkit-transition: border-color ease-in-out .15s, box-shadow ease-in-out .15s;
  -o-transition: border-color ease-in-out .15s, box-shadow ease-in-out .15s;
  transition: border-color ease-in-out .15s, box-shadow ease-in-out .15s;
}
.form-control:focus {
  border-color: #66afe9;
  outline: 0;
  -webkit-box-shadow: inset 0 1px 1px rgba(0,0,0,.075), 0 0 8px rgba(102, 175, 233, 0.6);
  box-shadow: inset 0 1px 1px rgba(0,0,0,.075), 0 0 8px rgba(102, 175, 233, 0.6);
}
.form-control::-moz-placeholder {
  color: #999;
  opacity: 1;
}
.form-control:-ms-input-placeholder {
  color: #999;
}
.form-control::-webkit-input-placeholder {
  color: #999;
}
.form-control::-ms-expand {
  border: 0;
  background-color: transparent;
}
.form-control[disabled],
.form-control[readonly],
fieldset[disabled] .form-control {
  background-color: #eeeeee;
  opacity: 1;
}
.form-control[disabled],
fieldset[disabled] .form-control {
  cursor: not-allowed;
}
textarea.form-control {
  height: auto;
}
input[type="search"] {
  -webkit-appearance: none;
}
@media screen and (-webkit-min-device-pixel-ratio: 0) {
  input[type="date"].form-control,
  input[type="time"].form-control,
  input[type="datetime-local"].form-control,
  input[type="month"].form-control {
    line-height: 32px;
  }
  input[type="date"].input-sm,
  input[type="time"].input-sm,
  input[type="datetime-local"].input-sm,
  input[type="month"].input-sm,
  .input-group-sm input[type="date"],
  .input-group-sm input[type="time"],
  .input-group-sm input[type="datetime-local"],
  .input-group-sm input[type="month"] {
    line-height: 30px;
  }
  input[type="date"].input-lg,
  input[type="time"].input-lg,
  input[type="datetime-local"].input-lg,
  input[type="month"].input-lg,
  .input-group-lg input[type="date"],
  .input-group-lg input[type="time"],
  .input-group-lg input[type="datetime-local"],
  .input-group-lg input[type="month"] {
    line-height: 45px;
  }
}
.form-group {
  margin-bottom: 15px;
}
.radio,
.checkbox {
  position: relative;
  display: block;
  margin-top: 10px;
  margin-bottom: 10px;
}
.radio label,
.checkbox label {
  min-height: 18px;
  padding-left: 20px;
  margin-bottom: 0;
  font-weight: normal;
  cursor: pointer;
}
.radio input[type="radio"],
.radio-inline input[type="radio"],
.checkbox input[type="checkbox"],
.checkbox-inline input[type="checkbox"] {
  position: absolute;
  margin-left: -20px;
  margin-top: 4px \9;
}
.radio + .radio,
.checkbox + .checkbox {
  margin-top: -5px;
}
.radio-inline,
.checkbox-inline {
  position: relative;
  display: inline-block;
  padding-left: 20px;
  margin-bottom: 0;
  vertical-align: middle;
  font-weight: normal;
  cursor: pointer;
}
.radio-inline + .radio-inline,
.checkbox-inline + .checkbox-inline {
  margin-top: 0;
  margin-left: 10px;
}
input[type="radio"][disabled],
input[type="checkbox"][disabled],
input[type="radio"].disabled,
input[type="checkbox"].disabled,
fieldset[disabled] input[type="radio"],
fieldset[disabled] input[type="checkbox"] {
  cursor: not-allowed;
}
.radio-inline.disabled,
.checkbox-inline.disabled,
fieldset[disabled] .radio-inline,
fieldset[disabled] .checkbox-inline {
  cursor: not-allowed;
}
.radio.disabled label,
.checkbox.disabled label,
fieldset[disabled] .radio label,
fieldset[disabled] .checkbox label {
  cursor: not-allowed;
}
.form-control-static {
  padding-top: 7px;
  padding-bottom: 7px;
  margin-bottom: 0;
  min-height: 31px;
}
.form-control-static.input-lg,
.form-control-static.input-sm {
  padding-left: 0;
  padding-right: 0;
}
.input-sm {
  height: 30px;
  padding: 5px 10px;
  font-size: 12px;
  line-height: 1.5;
  border-radius: 1px;
}
select.input-sm {
  height: 30px;
  line-height: 30px;
}
textarea.input-sm,
select[multiple].input-sm {
  height: auto;
}
.form-group-sm .form-control {
  height: 30px;
  padding: 5px 10px;
  font-size: 12px;
  line-height: 1.5;
  border-radius: 1px;
}
.form-group-sm select.form-control {
  height: 30px;
  line-height: 30px;
}
.form-group-sm textarea.form-control,
.form-group-sm select[multiple].form-control {
  height: auto;
}
.form-group-sm .form-control-static {
  height: 30px;
  min-height: 30px;
  padding: 6px 10px;
  font-size: 12px;
  line-height: 1.5;
}
.input-lg {
  height: 45px;
  padding: 10px 16px;
  font-size: 17px;
  line-height: 1.3333333;
  border-radius: 3px;
}
select.input-lg {
  height: 45px;
  line-height: 45px;
}
textarea.input-lg,
select[multiple].input-lg {
  height: auto;
}
.form-group-lg .form-control {
  height: 45px;
  padding: 10px 16px;
  font-size: 17px;
  line-height: 1.3333333;
  border-radius: 3px;
}
.form-group-lg select.form-control {
  height: 45px;
  line-height: 45px;
}
.form-group-lg textarea.form-control,
.form-group-lg select[multiple].form-control {
  height: auto;
}
.form-group-lg .form-control-static {
  height: 45px;
  min-height: 35px;
  padding: 11px 16px;
  font-size: 17px;
  line-height: 1.3333333;
}
.has-feedback {
  position: relative;
}
.has-feedback .form-control {
  padding-right: 40px;
}
.form-control-feedback {
  position: absolute;
  top: 0;
  right: 0;
  z-index: 2;
  display: block;
  width: 32px;
  height: 32px;
  line-height: 32px;
  text-align: center;
  pointer-events: none;
}
.input-lg + .form-control-feedback,
.input-group-lg + .form-control-feedback,
.form-group-lg .form-control + .form-control-feedback {
  width: 45px;
  height: 45px;
  line-height: 45px;
}
.input-sm + .form-control-feedback,
.input-group-sm + .form-control-feedback,
.form-group-sm .form-control + .form-control-feedback {
  width: 30px;
  height: 30px;
  line-height: 30px;
}
.has-success .help-block,
.has-success .control-label,
.has-success .radio,
.has-success .checkbox,
.has-success .radio-inline,
.has-success .checkbox-inline,
.has-success.radio label,
.has-success.checkbox label,
.has-success.radio-inline label,
.has-success.checkbox-inline label {
  color: #3c763d;
}
.has-success .form-control {
  border-color: #3c763d;
  -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
  box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
}
.has-success .form-control:focus {
  border-color: #2b542c;
  -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #67b168;
  box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #67b168;
}
.has-success .input-group-addon {
  color: #3c763d;
  border-color: #3c763d;
  background-color: #dff0d8;
}
.has-success .form-control-feedback {
  color: #3c763d;
}
.has-warning .help-block,
.has-warning .control-label,
.has-warning .radio,
.has-warning .checkbox,
.has-warning .radio-inline,
.has-warning .checkbox-inline,
.has-warning.radio label,
.has-warning.checkbox label,
.has-warning.radio-inline label,
.has-warning.checkbox-inline label {
  color: #8a6d3b;
}
.has-warning .form-control {
  border-color: #8a6d3b;
  -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
  box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
}
.has-warning .form-control:focus {
  border-color: #66512c;
  -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #c0a16b;
  box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #c0a16b;
}
.has-warning .input-group-addon {
  color: #8a6d3b;
  border-color: #8a6d3b;
  background-color: #fcf8e3;
}
.has-warning .form-control-feedback {
  color: #8a6d3b;
}
.has-error .help-block,
.has-error .control-label,
.has-error .radio,
.has-error .checkbox,
.has-error .radio-inline,
.has-error .checkbox-inline,
.has-error.radio label,
.has-error.checkbox label,
.has-error.radio-inline label,
.has-error.checkbox-inline label {
  color: #a94442;
}
.has-error .form-control {
  border-color: #a94442;
  -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
  box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
}
.has-error .form-control:focus {
  border-color: #843534;
  -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #ce8483;
  box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #ce8483;
}
.has-error .input-group-addon {
  color: #a94442;
  border-color: #a94442;
  background-color: #f2dede;
}
.has-error .form-control-feedback {
  color: #a94442;
}
.has-feedback label ~ .form-control-feedback {
  top: 23px;
}
.has-feedback label.sr-only ~ .form-control-feedback {
  top: 0;
}
.help-block {
  display: block;
  margin-top: 5px;
  margin-bottom: 10px;
  color: #404040;
}
@media (min-width: 768px) {
  .form-inline .form-group {
    display: inline-block;
    margin-bottom: 0;
    vertical-align: middle;
  }
  .form-inline .form-control {
    display: inline-block;
    width: auto;
    vertical-align: middle;
  }
  .form-inline .form-control-static {
    display: inline-block;
  }
  .form-inline .input-group {
    display: inline-table;
    vertical-align: middle;
  }
  .form-inline .input-group .input-group-addon,
  .form-inline .input-group .input-group-btn,
  .form-inline .input-group .form-control {
    width: auto;
  }
  .form-inline .input-group > .form-control {
    width: 100%;
  }
  .form-inline .control-label {
    margin-bottom: 0;
    vertical-align: middle;
  }
  .form-inline .radio,
  .form-inline .checkbox {
    display: inline-block;
    margin-top: 0;
    margin-bottom: 0;
    vertical-align: middle;
  }
  .form-inline .radio label,
  .form-inline .checkbox label {
    padding-left: 0;
  }
  .form-inline .radio input[type="radio"],
  .form-inline .checkbox input[type="checkbox"] {
    position: relative;
    margin-left: 0;
  }
  .form-inline .has-feedback .form-control-feedback {
    top: 0;
  }
}
.form-horizontal .radio,
.form-horizontal .checkbox,
.form-horizontal .radio-inline,
.form-horizontal .checkbox-inline {
  margin-top: 0;
  margin-bottom: 0;
  padding-top: 7px;
}
.form-horizontal .radio,
.form-horizontal .checkbox {
  min-height: 25px;
}
.form-horizontal .form-group {
  margin-left: 0px;
  margin-right: 0px;
}
@media (min-width: 768px) {
  .form-horizontal .control-label {
    text-align: right;
    margin-bottom: 0;
    padding-top: 7px;
  }
}
.form-horizontal .has-feedback .form-control-feedback {
  right: 0px;
}
@media (min-width: 768px) {
  .form-horizontal .form-group-lg .control-label {
    padding-top: 11px;
    font-size: 17px;
  }
}
@media (min-width: 768px) {
  .form-horizontal .form-group-sm .control-label {
    padding-top: 6px;
    font-size: 12px;
  }
}
.btn {
  display: inline-block;
  margin-bottom: 0;
  font-weight: normal;
  text-align: center;
  vertical-align: middle;
  touch-action: manipulation;
  cursor: pointer;
  background-image: none;
  border: 1px solid transparent;
  white-space: nowrap;
  padding: 6px 12px;
  font-size: 13px;
  line-height: 1.42857143;
  border-radius: 2px;
  -webkit-user-select: none;
  -moz-user-select: none;
  -ms-user-select: none;
  user-select: none;
}
.btn:focus,
.btn:active:focus,
.btn.active:focus,
.btn.focus,
.btn:active.focus,
.btn.active.focus {
  outline: 5px auto -webkit-focus-ring-color;
  outline-offset: -2px;
}
.btn:hover,
.btn:focus,
.btn.focus {
  color: #333;
  text-decoration: none;
}
.btn:active,
.btn.active {
  outline: 0;
  background-image: none;
  -webkit-box-shadow: inset 0 3px 5px rgba(0, 0, 0, 0.125);
  box-shadow: inset 0 3px 5px rgba(0, 0, 0, 0.125);
}
.btn.disabled,
.btn[disabled],
fieldset[disabled] .btn {
  cursor: not-allowed;
  opacity: 0.65;
  filter: alpha(opacity=65);
  -webkit-box-shadow: none;
  box-shadow: none;
}
a.btn.disabled,
fieldset[disabled] a.btn {
  pointer-events: none;
}
.btn-default {
  color: #333;
  background-color: #fff;
  border-color: #ccc;
}
.btn-default:focus,
.btn-default.focus {
  color: #333;
  background-color: #e6e6e6;
  border-color: #8c8c8c;
}
.btn-default:hover {
  color: #333;
  background-color: #e6e6e6;
  border-color: #adadad;
}
.btn-default:active,
.btn-default.active,
.open > .dropdown-toggle.btn-default {
  color: #333;
  background-color: #e6e6e6;
  border-color: #adadad;
}
.btn-default:active:hover,
.btn-default.active:hover,
.open > .dropdown-toggle.btn-default:hover,
.btn-default:active:focus,
.btn-default.active:focus,
.open > .dropdown-toggle.btn-default:focus,
.btn-default:active.focus,
.btn-default.active.focus,
.open > .dropdown-toggle.btn-default.focus {
  color: #333;
  background-color: #d4d4d4;
  border-color: #8c8c8c;
}
.btn-default:active,
.btn-default.active,
.open > .dropdown-toggle.btn-default {
  background-image: none;
}
.btn-default.disabled:hover,
.btn-default[disabled]:hover,
fieldset[disabled] .btn-default:hover,
.btn-default.disabled:focus,
.btn-default[disabled]:focus,
fieldset[disabled] .btn-default:focus,
.btn-default.disabled.focus,
.btn-default[disabled].focus,
fieldset[disabled] .btn-default.focus {
  background-color: #fff;
  border-color: #ccc;
}
.btn-default .badge {
  color: #fff;
  background-color: #333;
}
.btn-primary {
  color: #fff;
  background-color: #337ab7;
  border-color: #2e6da4;
}
.btn-primary:focus,
.btn-primary.focus {
  color: #fff;
  background-color: #286090;
  border-color: #122b40;
}
.btn-primary:hover {
  color: #fff;
  background-color: #286090;
  border-color: #204d74;
}
.btn-primary:active,
.btn-primary.active,
.open > .dropdown-toggle.btn-primary {
  color: #fff;
  background-color: #286090;
  border-color: #204d74;
}
.btn-primary:active:hover,
.btn-primary.active:hover,
.open > .dropdown-toggle.btn-primary:hover,
.btn-primary:active:focus,
.btn-primary.active:focus,
.open > .dropdown-toggle.btn-primary:focus,
.btn-primary:active.focus,
.btn-primary.active.focus,
.open > .dropdown-toggle.btn-primary.focus {
  color: #fff;
  background-color: #204d74;
  border-color: #122b40;
}
.btn-primary:active,
.btn-primary.active,
.open > .dropdown-toggle.btn-primary {
  background-image: none;
}
.btn-primary.disabled:hover,
.btn-primary[disabled]:hover,
fieldset[disabled] .btn-primary:hover,
.btn-primary.disabled:focus,
.btn-primary[disabled]:focus,
fieldset[disabled] .btn-primary:focus,
.btn-primary.disabled.focus,
.btn-primary[disabled].focus,
fieldset[disabled] .btn-primary.focus {
  background-color: #337ab7;
  border-color: #2e6da4;
}
.btn-primary .badge {
  color: #337ab7;
  background-color: #fff;
}
.btn-success {
  color: #fff;
  background-color: #5cb85c;
  border-color: #4cae4c;
}
.btn-success:focus,
.btn-success.focus {
  color: #fff;
  background-color: #449d44;
  border-color: #255625;
}
.btn-success:hover {
  color: #fff;
  background-color: #449d44;
  border-color: #398439;
}
.btn-success:active,
.btn-success.active,
.open > .dropdown-toggle.btn-success {
  color: #fff;
  background-color: #449d44;
  border-color: #398439;
}
.btn-success:active:hover,
.btn-success.active:hover,
.open > .dropdown-toggle.btn-success:hover,
.btn-success:active:focus,
.btn-success.active:focus,
.open > .dropdown-toggle.btn-success:focus,
.btn-success:active.focus,
.btn-success.active.focus,
.open > .dropdown-toggle.btn-success.focus {
  color: #fff;
  background-color: #398439;
  border-color: #255625;
}
.btn-success:active,
.btn-success.active,
.open > .dropdown-toggle.btn-success {
  background-image: none;
}
.btn-success.disabled:hover,
.btn-success[disabled]:hover,
fieldset[disabled] .btn-success:hover,
.btn-success.disabled:focus,
.btn-success[disabled]:focus,
fieldset[disabled] .btn-success:focus,
.btn-success.disabled.focus,
.btn-success[disabled].focus,
fieldset[disabled] .btn-success.focus {
  background-color: #5cb85c;
  border-color: #4cae4c;
}
.btn-success .badge {
  color: #5cb85c;
  background-color: #fff;
}
.btn-info {
  color: #fff;
  background-color: #5bc0de;
  border-color: #46b8da;
}
.btn-info:focus,
.btn-info.focus {
  color: #fff;
  background-color: #31b0d5;
  border-color: #1b6d85;
}
.btn-info:hover {
  color: #fff;
  background-color: #31b0d5;
  border-color: #269abc;
}
.btn-info:active,
.btn-info.active,
.open > .dropdown-toggle.btn-info {
  color: #fff;
  background-color: #31b0d5;
  border-color: #269abc;
}
.btn-info:active:hover,
.btn-info.active:hover,
.open > .dropdown-toggle.btn-info:hover,
.btn-info:active:focus,
.btn-info.active:focus,
.open > .dropdown-toggle.btn-info:focus,
.btn-info:active.focus,
.btn-info.active.focus,
.open > .dropdown-toggle.btn-info.focus {
  color: #fff;
  background-color: #269abc;
  border-color: #1b6d85;
}
.btn-info:active,
.btn-info.active,
.open > .dropdown-toggle.btn-info {
  background-image: none;
}
.btn-info.disabled:hover,
.btn-info[disabled]:hover,
fieldset[disabled] .btn-info:hover,
.btn-info.disabled:focus,
.btn-info[disabled]:focus,
fieldset[disabled] .btn-info:focus,
.btn-info.disabled.focus,
.btn-info[disabled].focus,
fieldset[disabled] .btn-info.focus {
  background-color: #5bc0de;
  border-color: #46b8da;
}
.btn-info .badge {
  color: #5bc0de;
  background-color: #fff;
}
.btn-warning {
  color: #fff;
  background-color: #f0ad4e;
  border-color: #eea236;
}
.btn-warning:focus,
.btn-warning.focus {
  color: #fff;
  background-color: #ec971f;
  border-color: #985f0d;
}
.btn-warning:hover {
  color: #fff;
  background-color: #ec971f;
  border-color: #d58512;
}
.btn-warning:active,
.btn-warning.active,
.open > .dropdown-toggle.btn-warning {
  color: #fff;
  background-color: #ec971f;
  border-color: #d58512;
}
.btn-warning:active:hover,
.btn-warning.active:hover,
.open > .dropdown-toggle.btn-warning:hover,
.btn-warning:active:focus,
.btn-warning.active:focus,
.open > .dropdown-toggle.btn-warning:focus,
.btn-warning:active.focus,
.btn-warning.active.focus,
.open > .dropdown-toggle.btn-warning.focus {
  color: #fff;
  background-color: #d58512;
  border-color: #985f0d;
}
.btn-warning:active,
.btn-warning.active,
.open > .dropdown-toggle.btn-warning {
  background-image: none;
}
.btn-warning.disabled:hover,
.btn-warning[disabled]:hover,
fieldset[disabled] .btn-warning:hover,
.btn-warning.disabled:focus,
.btn-warning[disabled]:focus,
fieldset[disabled] .btn-warning:focus,
.btn-warning.disabled.focus,
.btn-warning[disabled].focus,
fieldset[disabled] .btn-warning.focus {
  background-color: #f0ad4e;
  border-color: #eea236;
}
.btn-warning .badge {
  color: #f0ad4e;
  background-color: #fff;
}
.btn-danger {
  color: #fff;
  background-color: #d9534f;
  border-color: #d43f3a;
}
.btn-danger:focus,
.btn-danger.focus {
  color: #fff;
  background-color: #c9302c;
  border-color: #761c19;
}
.btn-danger:hover {
  color: #fff;
  background-color: #c9302c;
  border-color: #ac2925;
}
.btn-danger:active,
.btn-danger.active,
.open > .dropdown-toggle.btn-danger {
  color: #fff;
  background-color: #c9302c;
  border-color: #ac2925;
}
.btn-danger:active:hover,
.btn-danger.active:hover,
.open > .dropdown-toggle.btn-danger:hover,
.btn-danger:active:focus,
.btn-danger.active:focus,
.open > .dropdown-toggle.btn-danger:focus,
.btn-danger:active.focus,
.btn-danger.active.focus,
.open > .dropdown-toggle.btn-danger.focus {
  color: #fff;
  background-color: #ac2925;
  border-color: #761c19;
}
.btn-danger:active,
.btn-danger.active,
.open > .dropdown-toggle.btn-danger {
  background-image: none;
}
.btn-danger.disabled:hover,
.btn-danger[disabled]:hover,
fieldset[disabled] .btn-danger:hover,
.btn-danger.disabled:focus,
.btn-danger[disabled]:focus,
fieldset[disabled] .btn-danger:focus,
.btn-danger.disabled.focus,
.btn-danger[disabled].focus,
fieldset[disabled] .btn-danger.focus {
  background-color: #d9534f;
  border-color: #d43f3a;
}
.btn-danger .badge {
  color: #d9534f;
  background-color: #fff;
}
.btn-link {
  color: #337ab7;
  font-weight: normal;
  border-radius: 0;
}
.btn-link,
.btn-link:active,
.btn-link.active,
.btn-link[disabled],
fieldset[disabled] .btn-link {
  background-color: transparent;
  -webkit-box-shadow: none;
  box-shadow: none;
}
.btn-link,
.btn-link:hover,
.btn-link:focus,
.btn-link:active {
  border-color: transparent;
}
.btn-link:hover,
.btn-link:focus {
  color: #23527c;
  text-decoration: underline;
  background-color: transparent;
}
.btn-link[disabled]:hover,
fieldset[disabled] .btn-link:hover,
.btn-link[disabled]:focus,
fieldset[disabled] .btn-link:focus {
  color: #777777;
  text-decoration: none;
}
.btn-lg,
.btn-group-lg > .btn {
  padding: 10px 16px;
  font-size: 17px;
  line-height: 1.3333333;
  border-radius: 3px;
}
.btn-sm,
.btn-group-sm > .btn {
  padding: 5px 10px;
  font-size: 12px;
  line-height: 1.5;
  border-radius: 1px;
}
.btn-xs,
.btn-group-xs > .btn {
  padding: 1px 5px;
  font-size: 12px;
  line-height: 1.5;
  border-radius: 1px;
}
.btn-block {
  display: block;
  width: 100%;
}
.btn-block + .btn-block {
  margin-top: 5px;
}
input[type="submit"].btn-block,
input[type="reset"].btn-block,
input[type="button"].btn-block {
  width: 100%;
}
.fade {
  opacity: 0;
  -webkit-transition: opacity 0.15s linear;
  -o-transition: opacity 0.15s linear;
  transition: opacity 0.15s linear;
}
.fade.in {
  opacity: 1;
}
.collapse {
  display: none;
}
.collapse.in {
  display: block;
}
tr.collapse.in {
  display: table-row;
}
tbody.collapse.in {
  display: table-row-group;
}
.collapsing {
  position: relative;
  height: 0;
  overflow: hidden;
  -webkit-transition-property: height, visibility;
  transition-property: height, visibility;
  -webkit-transition-duration: 0.35s;
  transition-duration: 0.35s;
  -webkit-transition-timing-function: ease;
  transition-timing-function: ease;
}
.caret {
  display: inline-block;
  width: 0;
  height: 0;
  margin-left: 2px;
  vertical-align: middle;
  border-top: 4px dashed;
  border-top: 4px solid \9;
  border-right: 4px solid transparent;
  border-left: 4px solid transparent;
}
.dropup,
.dropdown {
  position: relative;
}
.dropdown-toggle:focus {
  outline: 0;
}
.dropdown-menu {
  position: absolute;
  top: 100%;
  left: 0;
  z-index: 1000;
  display: none;
  float: left;
  min-width: 160px;
  padding: 5px 0;
  margin: 2px 0 0;
  list-style: none;
  font-size: 13px;
  text-align: left;
  background-color: #fff;
  border: 1px solid #ccc;
  border: 1px solid rgba(0, 0, 0, 0.15);
  border-radius: 2px;
  -webkit-box-shadow: 0 6px 12px rgba(0, 0, 0, 0.175);
  box-shadow: 0 6px 12px rgba(0, 0, 0, 0.175);
  background-clip: padding-box;
}
.dropdown-menu.pull-right {
  right: 0;
  left: auto;
}
.dropdown-menu .divider {
  height: 1px;
  margin: 8px 0;
  overflow: hidden;
  background-color: #e5e5e5;
}
.dropdown-menu > li > a {
  display: block;
  padding: 3px 20px;
  clear: both;
  font-weight: normal;
  line-height: 1.42857143;
  color: #333333;
  white-space: nowrap;
}
.dropdown-menu > li > a:hover,
.dropdown-menu > li > a:focus {
  text-decoration: none;
  color: #262626;
  background-color: #f5f5f5;
}
.dropdown-menu > .active > a,
.dropdown-menu > .active > a:hover,
.dropdown-menu > .active > a:focus {
  color: #fff;
  text-decoration: none;
  outline: 0;
  background-color: #337ab7;
}
.dropdown-menu > .disabled > a,
.dropdown-menu > .disabled > a:hover,
.dropdown-menu > .disabled > a:focus {
  color: #777777;
}
.dropdown-menu > .disabled > a:hover,
.dropdown-menu > .disabled > a:focus {
  text-decoration: none;
  background-color: transparent;
  background-image: none;
  filter: progid:DXImageTransform.Microsoft.gradient(enabled = false);
  cursor: not-allowed;
}
.open > .dropdown-menu {
  display: block;
}
.open > a {
  outline: 0;
}
.dropdown-menu-right {
  left: auto;
  right: 0;
}
.dropdown-menu-left {
  left: 0;
  right: auto;
}
.dropdown-header {
  display: block;
  padding: 3px 20px;
  font-size: 12px;
  line-height: 1.42857143;
  color: #777777;
  white-space: nowrap;
}
.dropdown-backdrop {
  position: fixed;
  left: 0;
  right: 0;
  bottom: 0;
  top: 0;
  z-index: 990;
}
.pull-right > .dropdown-menu {
  right: 0;
  left: auto;
}
.dropup .caret,
.navbar-fixed-bottom .dropdown .caret {
  border-top: 0;
  border-bottom: 4px dashed;
  border-bottom: 4px solid \9;
  content: "";
}
.dropup .dropdown-menu,
.navbar-fixed-bottom .dropdown .dropdown-menu {
  top: auto;
  bottom: 100%;
  margin-bottom: 2px;
}
@media (min-width: 541px) {
  .navbar-right .dropdown-menu {
    left: auto;
    right: 0;
  }
  .navbar-right .dropdown-menu-left {
    left: 0;
    right: auto;
  }
}
.btn-group,
.btn-group-vertical {
  position: relative;
  display: inline-block;
  vertical-align: middle;
}
.btn-group > .btn,
.btn-group-vertical > .btn {
  position: relative;
  float: left;
}
.btn-group > .btn:hover,
.btn-group-vertical > .btn:hover,
.btn-group > .btn:focus,
.btn-group-vertical > .btn:focus,
.btn-group > .btn:active,
.btn-group-vertical > .btn:active,
.btn-group > .btn.active,
.btn-group-vertical > .btn.active {
  z-index: 2;
}
.btn-group .btn + .btn,
.btn-group .btn + .btn-group,
.btn-group .btn-group + .btn,
.btn-group .btn-group + .btn-group {
  margin-left: -1px;
}
.btn-toolbar {
  margin-left: -5px;
}
.btn-toolbar .btn,
.btn-toolbar .btn-group,
.btn-toolbar .input-group {
  float: left;
}
.btn-toolbar > .btn,
.btn-toolbar > .btn-group,
.btn-toolbar > .input-group {
  margin-left: 5px;
}
.btn-group > .btn:not(:first-child):not(:last-child):not(.dropdown-toggle) {
  border-radius: 0;
}
.btn-group > .btn:first-child {
  margin-left: 0;
}
.btn-group > .btn:first-child:not(:last-child):not(.dropdown-toggle) {
  border-bottom-right-radius: 0;
  border-top-right-radius: 0;
}
.btn-group > .btn:last-child:not(:first-child),
.btn-group > .dropdown-toggle:not(:first-child) {
  border-bottom-left-radius: 0;
  border-top-left-radius: 0;
}
.btn-group > .btn-group {
  float: left;
}
.btn-group > .btn-group:not(:first-child):not(:last-child) > .btn {
  border-radius: 0;
}
.btn-group > .btn-group:first-child:not(:last-child) > .btn:last-child,
.btn-group > .btn-group:first-child:not(:last-child) > .dropdown-toggle {
  border-bottom-right-radius: 0;
  border-top-right-radius: 0;
}
.btn-group > .btn-group:last-child:not(:first-child) > .btn:first-child {
  border-bottom-left-radius: 0;
  border-top-left-radius: 0;
}
.btn-group .dropdown-toggle:active,
.btn-group.open .dropdown-toggle {
  outline: 0;
}
.btn-group > .btn + .dropdown-toggle {
  padding-left: 8px;
  padding-right: 8px;
}
.btn-group > .btn-lg + .dropdown-toggle {
  padding-left: 12px;
  padding-right: 12px;
}
.btn-group.open .dropdown-toggle {
  -webkit-box-shadow: inset 0 3px 5px rgba(0, 0, 0, 0.125);
  box-shadow: inset 0 3px 5px rgba(0, 0, 0, 0.125);
}
.btn-group.open .dropdown-toggle.btn-link {
  -webkit-box-shadow: none;
  box-shadow: none;
}
.btn .caret {
  margin-left: 0;
}
.btn-lg .caret {
  border-width: 5px 5px 0;
  border-bottom-width: 0;
}
.dropup .btn-lg .caret {
  border-width: 0 5px 5px;
}
.btn-group-vertical > .btn,
.btn-group-vertical > .btn-group,
.btn-group-vertical > .btn-group > .btn {
  display: block;
  float: none;
  width: 100%;
  max-width: 100%;
}
.btn-group-vertical > .btn-group > .btn {
  float: none;
}
.btn-group-vertical > .btn + .btn,
.btn-group-vertical > .btn + .btn-group,
.btn-group-vertical > .btn-group + .btn,
.btn-group-vertical > .btn-group + .btn-group {
  margin-top: -1px;
  margin-left: 0;
}
.btn-group-vertical > .btn:not(:first-child):not(:last-child) {
  border-radius: 0;
}
.btn-group-vertical > .btn:first-child:not(:last-child) {
  border-top-right-radius: 2px;
  border-top-left-radius: 2px;
  border-bottom-right-radius: 0;
  border-bottom-left-radius: 0;
}
.btn-group-vertical > .btn:last-child:not(:first-child) {
  border-top-right-radius: 0;
  border-top-left-radius: 0;
  border-bottom-right-radius: 2px;
  border-bottom-left-radius: 2px;
}
.btn-group-vertical > .btn-group:not(:first-child):not(:last-child) > .btn {
  border-radius: 0;
}
.btn-group-vertical > .btn-group:first-child:not(:last-child) > .btn:last-child,
.btn-group-vertical > .btn-group:first-child:not(:last-child) > .dropdown-toggle {
  border-bottom-right-radius: 0;
  border-bottom-left-radius: 0;
}
.btn-group-vertical > .btn-group:last-child:not(:first-child) > .btn:first-child {
  border-top-right-radius: 0;
  border-top-left-radius: 0;
}
.btn-group-justified {
  display: table;
  width: 100%;
  table-layout: fixed;
  border-collapse: separate;
}
.btn-group-justified > .btn,
.btn-group-justified > .btn-group {
  float: none;
  display: table-cell;
  width: 1%;
}
.btn-group-justified > .btn-group .btn {
  width: 100%;
}
.btn-group-justified > .btn-group .dropdown-menu {
  left: auto;
}
[data-toggle="buttons"] > .btn input[type="radio"],
[data-toggle="buttons"] > .btn-group > .btn input[type="radio"],
[data-toggle="buttons"] > .btn input[type="checkbox"],
[data-toggle="buttons"] > .btn-group > .btn input[type="checkbox"] {
  position: absolute;
  clip: rect(0, 0, 0, 0);
  pointer-events: none;
}
.input-group {
  position: relative;
  display: table;
  border-collapse: separate;
}
.input-group[class*="col-"] {
  float: none;
  padding-left: 0;
  padding-right: 0;
}
.input-group .form-control {
  position: relative;
  z-index: 2;
  float: left;
  width: 100%;
  margin-bottom: 0;
}
.input-group .form-control:focus {
  z-index: 3;
}
.input-group-lg > .form-control,
.input-group-lg > .input-group-addon,
.input-group-lg > .input-group-btn > .btn {
  height: 45px;
  padding: 10px 16px;
  font-size: 17px;
  line-height: 1.3333333;
  border-radius: 3px;
}
select.input-group-lg > .form-control,
select.input-group-lg > .input-group-addon,
select.input-group-lg > .input-group-btn > .btn {
  height: 45px;
  line-height: 45px;
}
textarea.input-group-lg > .form-control,
textarea.input-group-lg > .input-group-addon,
textarea.input-group-lg > .input-group-btn > .btn,
select[multiple].input-group-lg > .form-control,
select[multiple].input-group-lg > .input-group-addon,
select[multiple].input-group-lg > .input-group-btn > .btn {
  height: auto;
}
.input-group-sm > .form-control,
.input-group-sm > .input-group-addon,
.input-group-sm > .input-group-btn > .btn {
  height: 30px;
  padding: 5px 10px;
  font-size: 12px;
  line-height: 1.5;
  border-radius: 1px;
}
select.input-group-sm > .form-control,
select.input-group-sm > .input-group-addon,
select.input-group-sm > .input-group-btn > .btn {
  height: 30px;
  line-height: 30px;
}
textarea.input-group-sm > .form-control,
textarea.input-group-sm > .input-group-addon,
textarea.input-group-sm > .input-group-btn > .btn,
select[multiple].input-group-sm > .form-control,
select[multiple].input-group-sm > .input-group-addon,
select[multiple].input-group-sm > .input-group-btn > .btn {
  height: auto;
}
.input-group-addon,
.input-group-btn,
.input-group .form-control {
  display: table-cell;
}
.input-group-addon:not(:first-child):not(:last-child),
.input-group-btn:not(:first-child):not(:last-child),
.input-group .form-control:not(:first-child):not(:last-child) {
  border-radius: 0;
}
.input-group-addon,
.input-group-btn {
  width: 1%;
  white-space: nowrap;
  vertical-align: middle;
}
.input-group-addon {
  padding: 6px 12px;
  font-size: 13px;
  font-weight: normal;
  line-height: 1;
  color: #555555;
  text-align: center;
  background-color: #eeeeee;
  border: 1px solid #ccc;
  border-radius: 2px;
}
.input-group-addon.input-sm {
  padding: 5px 10px;
  font-size: 12px;
  border-radius: 1px;
}
.input-group-addon.input-lg {
  padding: 10px 16px;
  font-size: 17px;
  border-radius: 3px;
}
.input-group-addon input[type="radio"],
.input-group-addon input[type="checkbox"] {
  margin-top: 0;
}
.input-group .form-control:first-child,
.input-group-addon:first-child,
.input-group-btn:first-child > .btn,
.input-group-btn:first-child > .btn-group > .btn,
.input-group-btn:first-child > .dropdown-toggle,
.input-group-btn:last-child > .btn:not(:last-child):not(.dropdown-toggle),
.input-group-btn:last-child > .btn-group:not(:last-child) > .btn {
  border-bottom-right-radius: 0;
  border-top-right-radius: 0;
}
.input-group-addon:first-child {
  border-right: 0;
}
.input-group .form-control:last-child,
.input-group-addon:last-child,
.input-group-btn:last-child > .btn,
.input-group-btn:last-child > .btn-group > .btn,
.input-group-btn:last-child > .dropdown-toggle,
.input-group-btn:first-child > .btn:not(:first-child),
.input-group-btn:first-child > .btn-group:not(:first-child) > .btn {
  border-bottom-left-radius: 0;
  border-top-left-radius: 0;
}
.input-group-addon:last-child {
  border-left: 0;
}
.input-group-btn {
  position: relative;
  font-size: 0;
  white-space: nowrap;
}
.input-group-btn > .btn {
  position: relative;
}
.input-group-btn > .btn + .btn {
  margin-left: -1px;
}
.input-group-btn > .btn:hover,
.input-group-btn > .btn:focus,
.input-group-btn > .btn:active {
  z-index: 2;
}
.input-group-btn:first-child > .btn,
.input-group-btn:first-child > .btn-group {
  margin-right: -1px;
}
.input-group-btn:last-child > .btn,
.input-group-btn:last-child > .btn-group {
  z-index: 2;
  margin-left: -1px;
}
.nav {
  margin-bottom: 0;
  padding-left: 0;
  list-style: none;
}
.nav > li {
  position: relative;
  display: block;
}
.nav > li > a {
  position: relative;
  display: block;
  padding: 10px 15px;
}
.nav > li > a:hover,
.nav > li > a:focus {
  text-decoration: none;
  background-color: #eeeeee;
}
.nav > li.disabled > a {
  color: #777777;
}
.nav > li.disabled > a:hover,
.nav > li.disabled > a:focus {
  color: #777777;
  text-decoration: none;
  background-color: transparent;
  cursor: not-allowed;
}
.nav .open > a,
.nav .open > a:hover,
.nav .open > a:focus {
  background-color: #eeeeee;
  border-color: #337ab7;
}
.nav .nav-divider {
  height: 1px;
  margin: 8px 0;
  overflow: hidden;
  background-color: #e5e5e5;
}
.nav > li > a > img {
  max-width: none;
}
.nav-tabs {
  border-bottom: 1px solid #ddd;
}
.nav-tabs > li {
  float: left;
  margin-bottom: -1px;
}
.nav-tabs > li > a {
  margin-right: 2px;
  line-height: 1.42857143;
  border: 1px solid transparent;
  border-radius: 2px 2px 0 0;
}
.nav-tabs > li > a:hover {
  border-color: #eeeeee #eeeeee #ddd;
}
.nav-tabs > li.active > a,
.nav-tabs > li.active > a:hover,
.nav-tabs > li.active > a:focus {
  color: #555555;
  background-color: #fff;
  border: 1px solid #ddd;
  border-bottom-color: transparent;
  cursor: default;
}
.nav-tabs.nav-justified {
  width: 100%;
  border-bottom: 0;
}
.nav-tabs.nav-justified > li {
  float: none;
}
.nav-tabs.nav-justified > li > a {
  text-align: center;
  margin-bottom: 5px;
}
.nav-tabs.nav-justified > .dropdown .dropdown-menu {
  top: auto;
  left: auto;
}
@media (min-width: 768px) {
  .nav-tabs.nav-justified > li {
    display: table-cell;
    width: 1%;
  }
  .nav-tabs.nav-justified > li > a {
    margin-bottom: 0;
  }
}
.nav-tabs.nav-justified > li > a {
  margin-right: 0;
  border-radius: 2px;
}
.nav-tabs.nav-justified > .active > a,
.nav-tabs.nav-justified > .active > a:hover,
.nav-tabs.nav-justified > .active > a:focus {
  border: 1px solid #ddd;
}
@media (min-width: 768px) {
  .nav-tabs.nav-justified > li > a {
    border-bottom: 1px solid #ddd;
    border-radius: 2px 2px 0 0;
  }
  .nav-tabs.nav-justified > .active > a,
  .nav-tabs.nav-justified > .active > a:hover,
  .nav-tabs.nav-justified > .active > a:focus {
    border-bottom-color: #fff;
  }
}
.nav-pills > li {
  float: left;
}
.nav-pills > li > a {
  border-radius: 2px;
}
.nav-pills > li + li {
  margin-left: 2px;
}
.nav-pills > li.active > a,
.nav-pills > li.active > a:hover,
.nav-pills > li.active > a:focus {
  color: #fff;
  background-color: #337ab7;
}
.nav-stacked > li {
  float: none;
}
.nav-stacked > li + li {
  margin-top: 2px;
  margin-left: 0;
}
.nav-justified {
  width: 100%;
}
.nav-justified > li {
  float: none;
}
.nav-justified > li > a {
  text-align: center;
  margin-bottom: 5px;
}
.nav-justified > .dropdown .dropdown-menu {
  top: auto;
  left: auto;
}
@media (min-width: 768px) {
  .nav-justified > li {
    display: table-cell;
    width: 1%;
  }
  .nav-justified > li > a {
    margin-bottom: 0;
  }
}
.nav-tabs-justified {
  border-bottom: 0;
}
.nav-tabs-justified > li > a {
  margin-right: 0;
  border-radius: 2px;
}
.nav-tabs-justified > .active > a,
.nav-tabs-justified > .active > a:hover,
.nav-tabs-justified > .active > a:focus {
  border: 1px solid #ddd;
}
@media (min-width: 768px) {
  .nav-tabs-justified > li > a {
    border-bottom: 1px solid #ddd;
    border-radius: 2px 2px 0 0;
  }
  .nav-tabs-justified > .active > a,
  .nav-tabs-justified > .active > a:hover,
  .nav-tabs-justified > .active > a:focus {
    border-bottom-color: #fff;
  }
}
.tab-content > .tab-pane {
  display: none;
}
.tab-content > .active {
  display: block;
}
.nav-tabs .dropdown-menu {
  margin-top: -1px;
  border-top-right-radius: 0;
  border-top-left-radius: 0;
}
.navbar {
  position: relative;
  min-height: 30px;
  margin-bottom: 18px;
  border: 1px solid transparent;
}
@media (min-width: 541px) {
  .navbar {
    border-radius: 2px;
  }
}
@media (min-width: 541px) {
  .navbar-header {
    float: left;
  }
}
.navbar-collapse {
  overflow-x: visible;
  padding-right: 0px;
  padding-left: 0px;
  border-top: 1px solid transparent;
  box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.1);
  -webkit-overflow-scrolling: touch;
}
.navbar-collapse.in {
  overflow-y: auto;
}
@media (min-width: 541px) {
  .navbar-collapse {
    width: auto;
    border-top: 0;
    box-shadow: none;
  }
  .navbar-collapse.collapse {
    display: block !important;
    height: auto !important;
    padding-bottom: 0;
    overflow: visible !important;
  }
  .navbar-collapse.in {
    overflow-y: visible;
  }
  .navbar-fixed-top .navbar-collapse,
  .navbar-static-top .navbar-collapse,
  .navbar-fixed-bottom .navbar-collapse {
    padding-left: 0;
    padding-right: 0;
  }
}
.navbar-fixed-top .navbar-collapse,
.navbar-fixed-bottom .navbar-collapse {
  max-height: 340px;
}
@media (max-device-width: 540px) and (orientation: landscape) {
  .navbar-fixed-top .navbar-collapse,
  .navbar-fixed-bottom .navbar-collapse {
    max-height: 200px;
  }
}
.container > .navbar-header,
.container-fluid > .navbar-header,
.container > .navbar-collapse,
.container-fluid > .navbar-collapse {
  margin-right: 0px;
  margin-left: 0px;
}
@media (min-width: 541px) {
  .container > .navbar-header,
  .container-fluid > .navbar-header,
  .container > .navbar-collapse,
  .container-fluid > .navbar-collapse {
    margin-right: 0;
    margin-left: 0;
  }
}
.navbar-static-top {
  z-index: 1000;
  border-width: 0 0 1px;
}
@media (min-width: 541px) {
  .navbar-static-top {
    border-radius: 0;
  }
}
.navbar-fixed-top,
.navbar-fixed-bottom {
  position: fixed;
  right: 0;
  left: 0;
  z-index: 1030;
}
@media (min-width: 541px) {
  .navbar-fixed-top,
  .navbar-fixed-bottom {
    border-radius: 0;
  }
}
.navbar-fixed-top {
  top: 0;
  border-width: 0 0 1px;
}
.navbar-fixed-bottom {
  bottom: 0;
  margin-bottom: 0;
  border-width: 1px 0 0;
}
.navbar-brand {
  float: left;
  padding: 6px 0px;
  font-size: 17px;
  line-height: 18px;
  height: 30px;
}
.navbar-brand:hover,
.navbar-brand:focus {
  text-decoration: none;
}
.navbar-brand > img {
  display: block;
}
@media (min-width: 541px) {
  .navbar > .container .navbar-brand,
  .navbar > .container-fluid .navbar-brand {
    margin-left: 0px;
  }
}
.navbar-toggle {
  position: relative;
  float: right;
  margin-right: 0px;
  padding: 9px 10px;
  margin-top: -2px;
  margin-bottom: -2px;
  background-color: transparent;
  background-image: none;
  border: 1px solid transparent;
  border-radius: 2px;
}
.navbar-toggle:focus {
  outline: 0;
}
.navbar-toggle .icon-bar {
  display: block;
  width: 22px;
  height: 2px;
  border-radius: 1px;
}
.navbar-toggle .icon-bar + .icon-bar {
  margin-top: 4px;
}
@media (min-width: 541px) {
  .navbar-toggle {
    display: none;
  }
}
.navbar-nav {
  margin: 3px 0px;
}
.navbar-nav > li > a {
  padding-top: 10px;
  padding-bottom: 10px;
  line-height: 18px;
}
@media (max-width: 540px) {
  .navbar-nav .open .dropdown-menu {
    position: static;
    float: none;
    width: auto;
    margin-top: 0;
    background-color: transparent;
    border: 0;
    box-shadow: none;
  }
  .navbar-nav .open .dropdown-menu > li > a,
  .navbar-nav .open .dropdown-menu .dropdown-header {
    padding: 5px 15px 5px 25px;
  }
  .navbar-nav .open .dropdown-menu > li > a {
    line-height: 18px;
  }
  .navbar-nav .open .dropdown-menu > li > a:hover,
  .navbar-nav .open .dropdown-menu > li > a:focus {
    background-image: none;
  }
}
@media (min-width: 541px) {
  .navbar-nav {
    float: left;
    margin: 0;
  }
  .navbar-nav > li {
    float: left;
  }
  .navbar-nav > li > a {
    padding-top: 6px;
    padding-bottom: 6px;
  }
}
.navbar-form {
  margin-left: 0px;
  margin-right: 0px;
  padding: 10px 0px;
  border-top: 1px solid transparent;
  border-bottom: 1px solid transparent;
  -webkit-box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.1), 0 1px 0 rgba(255, 255, 255, 0.1);
  box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.1), 0 1px 0 rgba(255, 255, 255, 0.1);
  margin-top: -1px;
  margin-bottom: -1px;
}
@media (min-width: 768px) {
  .navbar-form .form-group {
    display: inline-block;
    margin-bottom: 0;
    vertical-align: middle;
  }
  .navbar-form .form-control {
    display: inline-block;
    width: auto;
    vertical-align: middle;
  }
  .navbar-form .form-control-static {
    display: inline-block;
  }
  .navbar-form .input-group {
    display: inline-table;
    vertical-align: middle;
  }
  .navbar-form .input-group .input-group-addon,
  .navbar-form .input-group .input-group-btn,
  .navbar-form .input-group .form-control {
    width: auto;
  }
  .navbar-form .input-group > .form-control {
    width: 100%;
  }
  .navbar-form .control-label {
    margin-bottom: 0;
    vertical-align: middle;
  }
  .navbar-form .radio,
  .navbar-form .checkbox {
    display: inline-block;
    margin-top: 0;
    margin-bottom: 0;
    vertical-align: middle;
  }
  .navbar-form .radio label,
  .navbar-form .checkbox label {
    padding-left: 0;
  }
  .navbar-form .radio input[type="radio"],
  .navbar-form .checkbox input[type="checkbox"] {
    position: relative;
    margin-left: 0;
  }
  .navbar-form .has-feedback .form-control-feedback {
    top: 0;
  }
}
@media (max-width: 540px) {
  .navbar-form .form-group {
    margin-bottom: 5px;
  }
  .navbar-form .form-group:last-child {
    margin-bottom: 0;
  }
}
@media (min-width: 541px) {
  .navbar-form {
    width: auto;
    border: 0;
    margin-left: 0;
    margin-right: 0;
    padding-top: 0;
    padding-bottom: 0;
    -webkit-box-shadow: none;
    box-shadow: none;
  }
}
.navbar-nav > li > .dropdown-menu {
  margin-top: 0;
  border-top-right-radius: 0;
  border-top-left-radius: 0;
}
.navbar-fixed-bottom .navbar-nav > li > .dropdown-menu {
  margin-bottom: 0;
  border-top-right-radius: 2px;
  border-top-left-radius: 2px;
  border-bottom-right-radius: 0;
  border-bottom-left-radius: 0;
}
.navbar-btn {
  margin-top: -1px;
  margin-bottom: -1px;
}
.navbar-btn.btn-sm {
  margin-top: 0px;
  margin-bottom: 0px;
}
.navbar-btn.btn-xs {
  margin-top: 4px;
  margin-bottom: 4px;
}
.navbar-text {
  margin-top: 6px;
  margin-bottom: 6px;
}
@media (min-width: 541px) {
  .navbar-text {
    float: left;
    margin-left: 0px;
    margin-right: 0px;
  }
}
@media (min-width: 541px) {
  .navbar-left {
    float: left !important;
    float: left;
  }
  .navbar-right {
    float: right !important;
    float: right;
    margin-right: 0px;
  }
  .navbar-right ~ .navbar-right {
    margin-right: 0;
  }
}
.navbar-default {
  background-color: #f8f8f8;
  border-color: #e7e7e7;
}
.navbar-default .navbar-brand {
  color: #777;
}
.navbar-default .navbar-brand:hover,
.navbar-default .navbar-brand:focus {
  color: #5e5e5e;
  background-color: transparent;
}
.navbar-default .navbar-text {
  color: #777;
}
.navbar-default .navbar-nav > li > a {
  color: #777;
}
.navbar-default .navbar-nav > li > a:hover,
.navbar-default .navbar-nav > li > a:focus {
  color: #333;
  background-color: transparent;
}
.navbar-default .navbar-nav > .active > a,
.navbar-default .navbar-nav > .active > a:hover,
.navbar-default .navbar-nav > .active > a:focus {
  color: #555;
  background-color: #e7e7e7;
}
.navbar-default .navbar-nav > .disabled > a,
.navbar-default .navbar-nav > .disabled > a:hover,
.navbar-default .navbar-nav > .disabled > a:focus {
  color: #ccc;
  background-color: transparent;
}
.navbar-default .navbar-toggle {
  border-color: #ddd;
}
.navbar-default .navbar-toggle:hover,
.navbar-default .navbar-toggle:focus {
  background-color: #ddd;
}
.navbar-default .navbar-toggle .icon-bar {
  background-color: #888;
}
.navbar-default .navbar-collapse,
.navbar-default .navbar-form {
  border-color: #e7e7e7;
}
.navbar-default .navbar-nav > .open > a,
.navbar-default .navbar-nav > .open > a:hover,
.navbar-default .navbar-nav > .open > a:focus {
  background-color: #e7e7e7;
  color: #555;
}
@media (max-width: 540px) {
  .navbar-default .navbar-nav .open .dropdown-menu > li > a {
    color: #777;
  }
  .navbar-default .navbar-nav .open .dropdown-menu > li > a:hover,
  .navbar-default .navbar-nav .open .dropdown-menu > li > a:focus {
    color: #333;
    background-color: transparent;
  }
  .navbar-default .navbar-nav .open .dropdown-menu > .active > a,
  .navbar-default .navbar-nav .open .dropdown-menu > .active > a:hover,
  .navbar-default .navbar-nav .open .dropdown-menu > .active > a:focus {
    color: #555;
    background-color: #e7e7e7;
  }
  .navbar-default .navbar-nav .open .dropdown-menu > .disabled > a,
  .navbar-default .navbar-nav .open .dropdown-menu > .disabled > a:hover,
  .navbar-default .navbar-nav .open .dropdown-menu > .disabled > a:focus {
    color: #ccc;
    background-color: transparent;
  }
}
.navbar-default .navbar-link {
  color: #777;
}
.navbar-default .navbar-link:hover {
  color: #333;
}
.navbar-default .btn-link {
  color: #777;
}
.navbar-default .btn-link:hover,
.navbar-default .btn-link:focus {
  color: #333;
}
.navbar-default .btn-link[disabled]:hover,
fieldset[disabled] .navbar-default .btn-link:hover,
.navbar-default .btn-link[disabled]:focus,
fieldset[disabled] .navbar-default .btn-link:focus {
  color: #ccc;
}
.navbar-inverse {
  background-color: #222;
  border-color: #080808;
}
.navbar-inverse .navbar-brand {
  color: #9d9d9d;
}
.navbar-inverse .navbar-brand:hover,
.navbar-inverse .navbar-brand:focus {
  color: #fff;
  background-color: transparent;
}
.navbar-inverse .navbar-text {
  color: #9d9d9d;
}
.navbar-inverse .navbar-nav > li > a {
  color: #9d9d9d;
}
.navbar-inverse .navbar-nav > li > a:hover,
.navbar-inverse .navbar-nav > li > a:focus {
  color: #fff;
  background-color: transparent;
}
.navbar-inverse .navbar-nav > .active > a,
.navbar-inverse .navbar-nav > .active > a:hover,
.navbar-inverse .navbar-nav > .active > a:focus {
  color: #fff;
  background-color: #080808;
}
.navbar-inverse .navbar-nav > .disabled > a,
.navbar-inverse .navbar-nav > .disabled > a:hover,
.navbar-inverse .navbar-nav > .disabled > a:focus {
  color: #444;
  background-color: transparent;
}
.navbar-inverse .navbar-toggle {
  border-color: #333;
}
.navbar-inverse .navbar-toggle:hover,
.navbar-inverse .navbar-toggle:focus {
  background-color: #333;
}
.navbar-inverse .navbar-toggle .icon-bar {
  background-color: #fff;
}
.navbar-inverse .navbar-collapse,
.navbar-inverse .navbar-form {
  border-color: #101010;
}
.navbar-inverse .navbar-nav > .open > a,
.navbar-inverse .navbar-nav > .open > a:hover,
.navbar-inverse .navbar-nav > .open > a:focus {
  background-color: #080808;
  color: #fff;
}
@media (max-width: 540px) {
  .navbar-inverse .navbar-nav .open .dropdown-menu > .dropdown-header {
    border-color: #080808;
  }
  .navbar-inverse .navbar-nav .open .dropdown-menu .divider {
    background-color: #080808;
  }
  .navbar-inverse .navbar-nav .open .dropdown-menu > li > a {
    color: #9d9d9d;
  }
  .navbar-inverse .navbar-nav .open .dropdown-menu > li > a:hover,
  .navbar-inverse .navbar-nav .open .dropdown-menu > li > a:focus {
    color: #fff;
    background-color: transparent;
  }
  .navbar-inverse .navbar-nav .open .dropdown-menu > .active > a,
  .navbar-inverse .navbar-nav .open .dropdown-menu > .active > a:hover,
  .navbar-inverse .navbar-nav .open .dropdown-menu > .active > a:focus {
    color: #fff;
    background-color: #080808;
  }
  .navbar-inverse .navbar-nav .open .dropdown-menu > .disabled > a,
  .navbar-inverse .navbar-nav .open .dropdown-menu > .disabled > a:hover,
  .navbar-inverse .navbar-nav .open .dropdown-menu > .disabled > a:focus {
    color: #444;
    background-color: transparent;
  }
}
.navbar-inverse .navbar-link {
  color: #9d9d9d;
}
.navbar-inverse .navbar-link:hover {
  color: #fff;
}
.navbar-inverse .btn-link {
  color: #9d9d9d;
}
.navbar-inverse .btn-link:hover,
.navbar-inverse .btn-link:focus {
  color: #fff;
}
.navbar-inverse .btn-link[disabled]:hover,
fieldset[disabled] .navbar-inverse .btn-link:hover,
.navbar-inverse .btn-link[disabled]:focus,
fieldset[disabled] .navbar-inverse .btn-link:focus {
  color: #444;
}
.breadcrumb {
  padding: 8px 15px;
  margin-bottom: 18px;
  list-style: none;
  background-color: #f5f5f5;
  border-radius: 2px;
}
.breadcrumb > li {
  display: inline-block;
}
.breadcrumb > li + li:before {
  content: "/\00a0";
  padding: 0 5px;
  color: #5e5e5e;
}
.breadcrumb > .active {
  color: #777777;
}
.pagination {
  display: inline-block;
  padding-left: 0;
  margin: 18px 0;
  border-radius: 2px;
}
.pagination > li {
  display: inline;
}
.pagination > li > a,
.pagination > li > span {
  position: relative;
  float: left;
  padding: 6px 12px;
  line-height: 1.42857143;
  text-decoration: none;
  color: #337ab7;
  background-color: #fff;
  border: 1px solid #ddd;
  margin-left: -1px;
}
.pagination > li:first-child > a,
.pagination > li:first-child > span {
  margin-left: 0;
  border-bottom-left-radius: 2px;
  border-top-left-radius: 2px;
}
.pagination > li:last-child > a,
.pagination > li:last-child > span {
  border-bottom-right-radius: 2px;
  border-top-right-radius: 2px;
}
.pagination > li > a:hover,
.pagination > li > span:hover,
.pagination > li > a:focus,
.pagination > li > span:focus {
  z-index: 2;
  color: #23527c;
  background-color: #eeeeee;
  border-color: #ddd;
}
.pagination > .active > a,
.pagination > .active > span,
.pagination > .active > a:hover,
.pagination > .active > span:hover,
.pagination > .active > a:focus,
.pagination > .active > span:focus {
  z-index: 3;
  color: #fff;
  background-color: #337ab7;
  border-color: #337ab7;
  cursor: default;
}
.pagination > .disabled > span,
.pagination > .disabled > span:hover,
.pagination > .disabled > span:focus,
.pagination > .disabled > a,
.pagination > .disabled > a:hover,
.pagination > .disabled > a:focus {
  color: #777777;
  background-color: #fff;
  border-color: #ddd;
  cursor: not-allowed;
}
.pagination-lg > li > a,
.pagination-lg > li > span {
  padding: 10px 16px;
  font-size: 17px;
  line-height: 1.3333333;
}
.pagination-lg > li:first-child > a,
.pagination-lg > li:first-child > span {
  border-bottom-left-radius: 3px;
  border-top-left-radius: 3px;
}
.pagination-lg > li:last-child > a,
.pagination-lg > li:last-child > span {
  border-bottom-right-radius: 3px;
  border-top-right-radius: 3px;
}
.pagination-sm > li > a,
.pagination-sm > li > span {
  padding: 5px 10px;
  font-size: 12px;
  line-height: 1.5;
}
.pagination-sm > li:first-child > a,
.pagination-sm > li:first-child > span {
  border-bottom-left-radius: 1px;
  border-top-left-radius: 1px;
}
.pagination-sm > li:last-child > a,
.pagination-sm > li:last-child > span {
  border-bottom-right-radius: 1px;
  border-top-right-radius: 1px;
}
.pager {
  padding-left: 0;
  margin: 18px 0;
  list-style: none;
  text-align: center;
}
.pager li {
  display: inline;
}
.pager li > a,
.pager li > span {
  display: inline-block;
  padding: 5px 14px;
  background-color: #fff;
  border: 1px solid #ddd;
  border-radius: 15px;
}
.pager li > a:hover,
.pager li > a:focus {
  text-decoration: none;
  background-color: #eeeeee;
}
.pager .next > a,
.pager .next > span {
  float: right;
}
.pager .previous > a,
.pager .previous > span {
  float: left;
}
.pager .disabled > a,
.pager .disabled > a:hover,
.pager .disabled > a:focus,
.pager .disabled > span {
  color: #777777;
  background-color: #fff;
  cursor: not-allowed;
}
.label {
  display: inline;
  padding: .2em .6em .3em;
  font-size: 75%;
  font-weight: bold;
  line-height: 1;
  color: #fff;
  text-align: center;
  white-space: nowrap;
  vertical-align: baseline;
  border-radius: .25em;
}
a.label:hover,
a.label:focus {
  color: #fff;
  text-decoration: none;
  cursor: pointer;
}
.label:empty {
  display: none;
}
.btn .label {
  position: relative;
  top: -1px;
}
.label-default {
  background-color: #777777;
}
.label-default[href]:hover,
.label-default[href]:focus {
  background-color: #5e5e5e;
}
.label-primary {
  background-color: #337ab7;
}
.label-primary[href]:hover,
.label-primary[href]:focus {
  background-color: #286090;
}
.label-success {
  background-color: #5cb85c;
}
.label-success[href]:hover,
.label-success[href]:focus {
  background-color: #449d44;
}
.label-info {
  background-color: #5bc0de;
}
.label-info[href]:hover,
.label-info[href]:focus {
  background-color: #31b0d5;
}
.label-warning {
  background-color: #f0ad4e;
}
.label-warning[href]:hover,
.label-warning[href]:focus {
  background-color: #ec971f;
}
.label-danger {
  background-color: #d9534f;
}
.label-danger[href]:hover,
.label-danger[href]:focus {
  background-color: #c9302c;
}
.badge {
  display: inline-block;
  min-width: 10px;
  padding: 3px 7px;
  font-size: 12px;
  font-weight: bold;
  color: #fff;
  line-height: 1;
  vertical-align: middle;
  white-space: nowrap;
  text-align: center;
  background-color: #777777;
  border-radius: 10px;
}
.badge:empty {
  display: none;
}
.btn .badge {
  position: relative;
  top: -1px;
}
.btn-xs .badge,
.btn-group-xs > .btn .badge {
  top: 0;
  padding: 1px 5px;
}
a.badge:hover,
a.badge:focus {
  color: #fff;
  text-decoration: none;
  cursor: pointer;
}
.list-group-item.active > .badge,
.nav-pills > .active > a > .badge {
  color: #337ab7;
  background-color: #fff;
}
.list-group-item > .badge {
  float: right;
}
.list-group-item > .badge + .badge {
  margin-right: 5px;
}
.nav-pills > li > a > .badge {
  margin-left: 3px;
}
.jumbotron {
  padding-top: 30px;
  padding-bottom: 30px;
  margin-bottom: 30px;
  color: inherit;
  background-color: #eeeeee;
}
.jumbotron h1,
.jumbotron .h1 {
  color: inherit;
}
.jumbotron p {
  margin-bottom: 15px;
  font-size: 20px;
  font-weight: 200;
}
.jumbotron > hr {
  border-top-color: #d5d5d5;
}
.container .jumbotron,
.container-fluid .jumbotron {
  border-radius: 3px;
  padding-left: 0px;
  padding-right: 0px;
}
.jumbotron .container {
  max-width: 100%;
}
@media screen and (min-width: 768px) {
  .jumbotron {
    padding-top: 48px;
    padding-bottom: 48px;
  }
  .container .jumbotron,
  .container-fluid .jumbotron {
    padding-left: 60px;
    padding-right: 60px;
  }
  .jumbotron h1,
  .jumbotron .h1 {
    font-size: 59px;
  }
}
.thumbnail {
  display: block;
  padding: 4px;
  margin-bottom: 18px;
  line-height: 1.42857143;
  background-color: #fff;
  border: 1px solid #ddd;
  border-radius: 2px;
  -webkit-transition: border 0.2s ease-in-out;
  -o-transition: border 0.2s ease-in-out;
  transition: border 0.2s ease-in-out;
}
.thumbnail > img,
.thumbnail a > img {
  margin-left: auto;
  margin-right: auto;
}
a.thumbnail:hover,
a.thumbnail:focus,
a.thumbnail.active {
  border-color: #337ab7;
}
.thumbnail .caption {
  padding: 9px;
  color: #000;
}
.alert {
  padding: 15px;
  margin-bottom: 18px;
  border: 1px solid transparent;
  border-radius: 2px;
}
.alert h4 {
  margin-top: 0;
  color: inherit;
}
.alert .alert-link {
  font-weight: bold;
}
.alert > p,
.alert > ul {
  margin-bottom: 0;
}
.alert > p + p {
  margin-top: 5px;
}
.alert-dismissable,
.alert-dismissible {
  padding-right: 35px;
}
.alert-dismissable .close,
.alert-dismissible .close {
  position: relative;
  top: -2px;
  right: -21px;
  color: inherit;
}
.alert-success {
  background-color: #dff0d8;
  border-color: #d6e9c6;
  color: #3c763d;
}
.alert-success hr {
  border-top-color: #c9e2b3;
}
.alert-success .alert-link {
  color: #2b542c;
}
.alert-info {
  background-color: #d9edf7;
  border-color: #bce8f1;
  color: #31708f;
}
.alert-info hr {
  border-top-color: #a6e1ec;
}
.alert-info .alert-link {
  color: #245269;
}
.alert-warning {
  background-color: #fcf8e3;
  border-color: #faebcc;
  color: #8a6d3b;
}
.alert-warning hr {
  border-top-color: #f7e1b5;
}
.alert-warning .alert-link {
  color: #66512c;
}
.alert-danger {
  background-color: #f2dede;
  border-color: #ebccd1;
  color: #a94442;
}
.alert-danger hr {
  border-top-color: #e4b9c0;
}
.alert-danger .alert-link {
  color: #843534;
}
@-webkit-keyframes progress-bar-stripes {
  from {
    background-position: 40px 0;
  }
  to {
    background-position: 0 0;
  }
}
@keyframes progress-bar-stripes {
  from {
    background-position: 40px 0;
  }
  to {
    background-position: 0 0;
  }
}
.progress {
  overflow: hidden;
  height: 18px;
  margin-bottom: 18px;
  background-color: #f5f5f5;
  border-radius: 2px;
  -webkit-box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.1);
  box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.1);
}
.progress-bar {
  float: left;
  width: 0%;
  height: 100%;
  font-size: 12px;
  line-height: 18px;
  color: #fff;
  text-align: center;
  background-color: #337ab7;
  -webkit-box-shadow: inset 0 -1px 0 rgba(0, 0, 0, 0.15);
  box-shadow: inset 0 -1px 0 rgba(0, 0, 0, 0.15);
  -webkit-transition: width 0.6s ease;
  -o-transition: width 0.6s ease;
  transition: width 0.6s ease;
}
.progress-striped .progress-bar,
.progress-bar-striped {
  background-image: -webkit-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
  background-image: -o-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
  background-image: linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
  background-size: 40px 40px;
}
.progress.active .progress-bar,
.progress-bar.active {
  -webkit-animation: progress-bar-stripes 2s linear infinite;
  -o-animation: progress-bar-stripes 2s linear infinite;
  animation: progress-bar-stripes 2s linear infinite;
}
.progress-bar-success {
  background-color: #5cb85c;
}
.progress-striped .progress-bar-success {
  background-image: -webkit-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
  background-image: -o-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
  background-image: linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
}
.progress-bar-info {
  background-color: #5bc0de;
}
.progress-striped .progress-bar-info {
  background-image: -webkit-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
  background-image: -o-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
  background-image: linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
}
.progress-bar-warning {
  background-color: #f0ad4e;
}
.progress-striped .progress-bar-warning {
  background-image: -webkit-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
  background-image: -o-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
  background-image: linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
}
.progress-bar-danger {
  background-color: #d9534f;
}
.progress-striped .progress-bar-danger {
  background-image: -webkit-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
  background-image: -o-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
  background-image: linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
}
.media {
  margin-top: 15px;
}
.media:first-child {
  margin-top: 0;
}
.media,
.media-body {
  zoom: 1;
  overflow: hidden;
}
.media-body {
  width: 10000px;
}
.media-object {
  display: block;
}
.media-object.img-thumbnail {
  max-width: none;
}
.media-right,
.media > .pull-right {
  padding-left: 10px;
}
.media-left,
.media > .pull-left {
  padding-right: 10px;
}
.media-left,
.media-right,
.media-body {
  display: table-cell;
  vertical-align: top;
}
.media-middle {
  vertical-align: middle;
}
.media-bottom {
  vertical-align: bottom;
}
.media-heading {
  margin-top: 0;
  margin-bottom: 5px;
}
.media-list {
  padding-left: 0;
  list-style: none;
}
.list-group {
  margin-bottom: 20px;
  padding-left: 0;
}
.list-group-item {
  position: relative;
  display: block;
  padding: 10px 15px;
  margin-bottom: -1px;
  background-color: #fff;
  border: 1px solid #ddd;
}
.list-group-item:first-child {
  border-top-right-radius: 2px;
  border-top-left-radius: 2px;
}
.list-group-item:last-child {
  margin-bottom: 0;
  border-bottom-right-radius: 2px;
  border-bottom-left-radius: 2px;
}
a.list-group-item,
button.list-group-item {
  color: #555;
}
a.list-group-item .list-group-item-heading,
button.list-group-item .list-group-item-heading {
  color: #333;
}
a.list-group-item:hover,
button.list-group-item:hover,
a.list-group-item:focus,
button.list-group-item:focus {
  text-decoration: none;
  color: #555;
  background-color: #f5f5f5;
}
button.list-group-item {
  width: 100%;
  text-align: left;
}
.list-group-item.disabled,
.list-group-item.disabled:hover,
.list-group-item.disabled:focus {
  background-color: #eeeeee;
  color: #777777;
  cursor: not-allowed;
}
.list-group-item.disabled .list-group-item-heading,
.list-group-item.disabled:hover .list-group-item-heading,
.list-group-item.disabled:focus .list-group-item-heading {
  color: inherit;
}
.list-group-item.disabled .list-group-item-text,
.list-group-item.disabled:hover .list-group-item-text,
.list-group-item.disabled:focus .list-group-item-text {
  color: #777777;
}
.list-group-item.active,
.list-group-item.active:hover,
.list-group-item.active:focus {
  z-index: 2;
  color: #fff;
  background-color: #337ab7;
  border-color: #337ab7;
}
.list-group-item.active .list-group-item-heading,
.list-group-item.active:hover .list-group-item-heading,
.list-group-item.active:focus .list-group-item-heading,
.list-group-item.active .list-group-item-heading > small,
.list-group-item.active:hover .list-group-item-heading > small,
.list-group-item.active:focus .list-group-item-heading > small,
.list-group-item.active .list-group-item-heading > .small,
.list-group-item.active:hover .list-group-item-heading > .small,
.list-group-item.active:focus .list-group-item-heading > .small {
  color: inherit;
}
.list-group-item.active .list-group-item-text,
.list-group-item.active:hover .list-group-item-text,
.list-group-item.active:focus .list-group-item-text {
  color: #c7ddef;
}
.list-group-item-success {
  color: #3c763d;
  background-color: #dff0d8;
}
a.list-group-item-success,
button.list-group-item-success {
  color: #3c763d;
}
a.list-group-item-success .list-group-item-heading,
button.list-group-item-success .list-group-item-heading {
  color: inherit;
}
a.list-group-item-success:hover,
button.list-group-item-success:hover,
a.list-group-item-success:focus,
button.list-group-item-success:focus {
  color: #3c763d;
  background-color: #d0e9c6;
}
a.list-group-item-success.active,
button.list-group-item-success.active,
a.list-group-item-success.active:hover,
button.list-group-item-success.active:hover,
a.list-group-item-success.active:focus,
button.list-group-item-success.active:focus {
  color: #fff;
  background-color: #3c763d;
  border-color: #3c763d;
}
.list-group-item-info {
  color: #31708f;
  background-color: #d9edf7;
}
a.list-group-item-info,
button.list-group-item-info {
  color: #31708f;
}
a.list-group-item-info .list-group-item-heading,
button.list-group-item-info .list-group-item-heading {
  color: inherit;
}
a.list-group-item-info:hover,
button.list-group-item-info:hover,
a.list-group-item-info:focus,
button.list-group-item-info:focus {
  color: #31708f;
  background-color: #c4e3f3;
}
a.list-group-item-info.active,
button.list-group-item-info.active,
a.list-group-item-info.active:hover,
button.list-group-item-info.active:hover,
a.list-group-item-info.active:focus,
button.list-group-item-info.active:focus {
  color: #fff;
  background-color: #31708f;
  border-color: #31708f;
}
.list-group-item-warning {
  color: #8a6d3b;
  background-color: #fcf8e3;
}
a.list-group-item-warning,
button.list-group-item-warning {
  color: #8a6d3b;
}
a.list-group-item-warning .list-group-item-heading,
button.list-group-item-warning .list-group-item-heading {
  color: inherit;
}
a.list-group-item-warning:hover,
button.list-group-item-warning:hover,
a.list-group-item-warning:focus,
button.list-group-item-warning:focus {
  color: #8a6d3b;
  background-color: #faf2cc;
}
a.list-group-item-warning.active,
button.list-group-item-warning.active,
a.list-group-item-warning.active:hover,
button.list-group-item-warning.active:hover,
a.list-group-item-warning.active:focus,
button.list-group-item-warning.active:focus {
  color: #fff;
  background-color: #8a6d3b;
  border-color: #8a6d3b;
}
.list-group-item-danger {
  color: #a94442;
  background-color: #f2dede;
}
a.list-group-item-danger,
button.list-group-item-danger {
  color: #a94442;
}
a.list-group-item-danger .list-group-item-heading,
button.list-group-item-danger .list-group-item-heading {
  color: inherit;
}
a.list-group-item-danger:hover,
button.list-group-item-danger:hover,
a.list-group-item-danger:focus,
button.list-group-item-danger:focus {
  color: #a94442;
  background-color: #ebcccc;
}
a.list-group-item-danger.active,
button.list-group-item-danger.active,
a.list-group-item-danger.active:hover,
button.list-group-item-danger.active:hover,
a.list-group-item-danger.active:focus,
button.list-group-item-danger.active:focus {
  color: #fff;
  background-color: #a94442;
  border-color: #a94442;
}
.list-group-item-heading {
  margin-top: 0;
  margin-bottom: 5px;
}
.list-group-item-text {
  margin-bottom: 0;
  line-height: 1.3;
}
.panel {
  margin-bottom: 18px;
  background-color: #fff;
  border: 1px solid transparent;
  border-radius: 2px;
  -webkit-box-shadow: 0 1px 1px rgba(0, 0, 0, 0.05);
  box-shadow: 0 1px 1px rgba(0, 0, 0, 0.05);
}
.panel-body {
  padding: 15px;
}
.panel-heading {
  padding: 10px 15px;
  border-bottom: 1px solid transparent;
  border-top-right-radius: 1px;
  border-top-left-radius: 1px;
}
.panel-heading > .dropdown .dropdown-toggle {
  color: inherit;
}
.panel-title {
  margin-top: 0;
  margin-bottom: 0;
  font-size: 15px;
  color: inherit;
}
.panel-title > a,
.panel-title > small,
.panel-title > .small,
.panel-title > small > a,
.panel-title > .small > a {
  color: inherit;
}
.panel-footer {
  padding: 10px 15px;
  background-color: #f5f5f5;
  border-top: 1px solid #ddd;
  border-bottom-right-radius: 1px;
  border-bottom-left-radius: 1px;
}
.panel > .list-group,
.panel > .panel-collapse > .list-group {
  margin-bottom: 0;
}
.panel > .list-group .list-group-item,
.panel > .panel-collapse > .list-group .list-group-item {
  border-width: 1px 0;
  border-radius: 0;
}
.panel > .list-group:first-child .list-group-item:first-child,
.panel > .panel-collapse > .list-group:first-child .list-group-item:first-child {
  border-top: 0;
  border-top-right-radius: 1px;
  border-top-left-radius: 1px;
}
.panel > .list-group:last-child .list-group-item:last-child,
.panel > .panel-collapse > .list-group:last-child .list-group-item:last-child {
  border-bottom: 0;
  border-bottom-right-radius: 1px;
  border-bottom-left-radius: 1px;
}
.panel > .panel-heading + .panel-collapse > .list-group .list-group-item:first-child {
  border-top-right-radius: 0;
  border-top-left-radius: 0;
}
.panel-heading + .list-group .list-group-item:first-child {
  border-top-width: 0;
}
.list-group + .panel-footer {
  border-top-width: 0;
}
.panel > .table,
.panel > .table-responsive > .table,
.panel > .panel-collapse > .table {
  margin-bottom: 0;
}
.panel > .table caption,
.panel > .table-responsive > .table caption,
.panel > .panel-collapse > .table caption {
  padding-left: 15px;
  padding-right: 15px;
}
.panel > .table:first-child,
.panel > .table-responsive:first-child > .table:first-child {
  border-top-right-radius: 1px;
  border-top-left-radius: 1px;
}
.panel > .table:first-child > thead:first-child > tr:first-child,
.panel > .table-responsive:first-child > .table:first-child > thead:first-child > tr:first-child,
.panel > .table:first-child > tbody:first-child > tr:first-child,
.panel > .table-responsive:first-child > .table:first-child > tbody:first-child > tr:first-child {
  border-top-left-radius: 1px;
  border-top-right-radius: 1px;
}
.panel > .table:first-child > thead:first-child > tr:first-child td:first-child,
.panel > .table-responsive:first-child > .table:first-child > thead:first-child > tr:first-child td:first-child,
.panel > .table:first-child > tbody:first-child > tr:first-child td:first-child,
.panel > .table-responsive:first-child > .table:first-child > tbody:first-child > tr:first-child td:first-child,
.panel > .table:first-child > thead:first-child > tr:first-child th:first-child,
.panel > .table-responsive:first-child > .table:first-child > thead:first-child > tr:first-child th:first-child,
.panel > .table:first-child > tbody:first-child > tr:first-child th:first-child,
.panel > .table-responsive:first-child > .table:first-child > tbody:first-child > tr:first-child th:first-child {
  border-top-left-radius: 1px;
}
.panel > .table:first-child > thead:first-child > tr:first-child td:last-child,
.panel > .table-responsive:first-child > .table:first-child > thead:first-child > tr:first-child td:last-child,
.panel > .table:first-child > tbody:first-child > tr:first-child td:last-child,
.panel > .table-responsive:first-child > .table:first-child > tbody:first-child > tr:first-child td:last-child,
.panel > .table:first-child > thead:first-child > tr:first-child th:last-child,
.panel > .table-responsive:first-child > .table:first-child > thead:first-child > tr:first-child th:last-child,
.panel > .table:first-child > tbody:first-child > tr:first-child th:last-child,
.panel > .table-responsive:first-child > .table:first-child > tbody:first-child > tr:first-child th:last-child {
  border-top-right-radius: 1px;
}
.panel > .table:last-child,
.panel > .table-responsive:last-child > .table:last-child {
  border-bottom-right-radius: 1px;
  border-bottom-left-radius: 1px;
}
.panel > .table:last-child > tbody:last-child > tr:last-child,
.panel > .table-responsive:last-child > .table:last-child > tbody:last-child > tr:last-child,
.panel > .table:last-child > tfoot:last-child > tr:last-child,
.panel > .table-responsive:last-child > .table:last-child > tfoot:last-child > tr:last-child {
  border-bottom-left-radius: 1px;
  border-bottom-right-radius: 1px;
}
.panel > .table:last-child > tbody:last-child > tr:last-child td:first-child,
.panel > .table-responsive:last-child > .table:last-child > tbody:last-child > tr:last-child td:first-child,
.panel > .table:last-child > tfoot:last-child > tr:last-child td:first-child,
.panel > .table-responsive:last-child > .table:last-child > tfoot:last-child > tr:last-child td:first-child,
.panel > .table:last-child > tbody:last-child > tr:last-child th:first-child,
.panel > .table-responsive:last-child > .table:last-child > tbody:last-child > tr:last-child th:first-child,
.panel > .table:last-child > tfoot:last-child > tr:last-child th:first-child,
.panel > .table-responsive:last-child > .table:last-child > tfoot:last-child > tr:last-child th:first-child {
  border-bottom-left-radius: 1px;
}
.panel > .table:last-child > tbody:last-child > tr:last-child td:last-child,
.panel > .table-responsive:last-child > .table:last-child > tbody:last-child > tr:last-child td:last-child,
.panel > .table:last-child > tfoot:last-child > tr:last-child td:last-child,
.panel > .table-responsive:last-child > .table:last-child > tfoot:last-child > tr:last-child td:last-child,
.panel > .table:last-child > tbody:last-child > tr:last-child th:last-child,
.panel > .table-responsive:last-child > .table:last-child > tbody:last-child > tr:last-child th:last-child,
.panel > .table:last-child > tfoot:last-child > tr:last-child th:last-child,
.panel > .table-responsive:last-child > .table:last-child > tfoot:last-child > tr:last-child th:last-child {
  border-bottom-right-radius: 1px;
}
.panel > .panel-body + .table,
.panel > .panel-body + .table-responsive,
.panel > .table + .panel-body,
.panel > .table-responsive + .panel-body {
  border-top: 1px solid #ddd;
}
.panel > .table > tbody:first-child > tr:first-child th,
.panel > .table > tbody:first-child > tr:first-child td {
  border-top: 0;
}
.panel > .table-bordered,
.panel > .table-responsive > .table-bordered {
  border: 0;
}
.panel > .table-bordered > thead > tr > th:first-child,
.panel > .table-responsive > .table-bordered > thead > tr > th:first-child,
.panel > .table-bordered > tbody > tr > th:first-child,
.panel > .table-responsive > .table-bordered > tbody > tr > th:first-child,
.panel > .table-bordered > tfoot > tr > th:first-child,
.panel > .table-responsive > .table-bordered > tfoot > tr > th:first-child,
.panel > .table-bordered > thead > tr > td:first-child,
.panel > .table-responsive > .table-bordered > thead > tr > td:first-child,
.panel > .table-bordered > tbody > tr > td:first-child,
.panel > .table-responsive > .table-bordered > tbody > tr > td:first-child,
.panel > .table-bordered > tfoot > tr > td:first-child,
.panel > .table-responsive > .table-bordered > tfoot > tr > td:first-child {
  border-left: 0;
}
.panel > .table-bordered > thead > tr > th:last-child,
.panel > .table-responsive > .table-bordered > thead > tr > th:last-child,
.panel > .table-bordered > tbody > tr > th:last-child,
.panel > .table-responsive > .table-bordered > tbody > tr > th:last-child,
.panel > .table-bordered > tfoot > tr > th:last-child,
.panel > .table-responsive > .table-bordered > tfoot > tr > th:last-child,
.panel > .table-bordered > thead > tr > td:last-child,
.panel > .table-responsive > .table-bordered > thead > tr > td:last-child,
.panel > .table-bordered > tbody > tr > td:last-child,
.panel > .table-responsive > .table-bordered > tbody > tr > td:last-child,
.panel > .table-bordered > tfoot > tr > td:last-child,
.panel > .table-responsive > .table-bordered > tfoot > tr > td:last-child {
  border-right: 0;
}
.panel > .table-bordered > thead > tr:first-child > td,
.panel > .table-responsive > .table-bordered > thead > tr:first-child > td,
.panel > .table-bordered > tbody > tr:first-child > td,
.panel > .table-responsive > .table-bordered > tbody > tr:first-child > td,
.panel > .table-bordered > thead > tr:first-child > th,
.panel > .table-responsive > .table-bordered > thead > tr:first-child > th,
.panel > .table-bordered > tbody > tr:first-child > th,
.panel > .table-responsive > .table-bordered > tbody > tr:first-child > th {
  border-bottom: 0;
}
.panel > .table-bordered > tbody > tr:last-child > td,
.panel > .table-responsive > .table-bordered > tbody > tr:last-child > td,
.panel > .table-bordered > tfoot > tr:last-child > td,
.panel > .table-responsive > .table-bordered > tfoot > tr:last-child > td,
.panel > .table-bordered > tbody > tr:last-child > th,
.panel > .table-responsive > .table-bordered > tbody > tr:last-child > th,
.panel > .table-bordered > tfoot > tr:last-child > th,
.panel > .table-responsive > .table-bordered > tfoot > tr:last-child > th {
  border-bottom: 0;
}
.panel > .table-responsive {
  border: 0;
  margin-bottom: 0;
}
.panel-group {
  margin-bottom: 18px;
}
.panel-group .panel {
  margin-bottom: 0;
  border-radius: 2px;
}
.panel-group .panel + .panel {
  margin-top: 5px;
}
.panel-group .panel-heading {
  border-bottom: 0;
}
.panel-group .panel-heading + .panel-collapse > .panel-body,
.panel-group .panel-heading + .panel-collapse > .list-group {
  border-top: 1px solid #ddd;
}
.panel-group .panel-footer {
  border-top: 0;
}
.panel-group .panel-footer + .panel-collapse .panel-body {
  border-bottom: 1px solid #ddd;
}
.panel-default {
  border-color: #ddd;
}
.panel-default > .panel-heading {
  color: #333333;
  background-color: #f5f5f5;
  border-color: #ddd;
}
.panel-default > .panel-heading + .panel-collapse > .panel-body {
  border-top-color: #ddd;
}
.panel-default > .panel-heading .badge {
  color: #f5f5f5;
  background-color: #333333;
}
.panel-default > .panel-footer + .panel-collapse > .panel-body {
  border-bottom-color: #ddd;
}
.panel-primary {
  border-color: #337ab7;
}
.panel-primary > .panel-heading {
  color: #fff;
  background-color: #337ab7;
  border-color: #337ab7;
}
.panel-primary > .panel-heading + .panel-collapse > .panel-body {
  border-top-color: #337ab7;
}
.panel-primary > .panel-heading .badge {
  color: #337ab7;
  background-color: #fff;
}
.panel-primary > .panel-footer + .panel-collapse > .panel-body {
  border-bottom-color: #337ab7;
}
.panel-success {
  border-color: #d6e9c6;
}
.panel-success > .panel-heading {
  color: #3c763d;
  background-color: #dff0d8;
  border-color: #d6e9c6;
}
.panel-success > .panel-heading + .panel-collapse > .panel-body {
  border-top-color: #d6e9c6;
}
.panel-success > .panel-heading .badge {
  color: #dff0d8;
  background-color: #3c763d;
}
.panel-success > .panel-footer + .panel-collapse > .panel-body {
  border-bottom-color: #d6e9c6;
}
.panel-info {
  border-color: #bce8f1;
}
.panel-info > .panel-heading {
  color: #31708f;
  background-color: #d9edf7;
  border-color: #bce8f1;
}
.panel-info > .panel-heading + .panel-collapse > .panel-body {
  border-top-color: #bce8f1;
}
.panel-info > .panel-heading .badge {
  color: #d9edf7;
  background-color: #31708f;
}
.panel-info > .panel-footer + .panel-collapse > .panel-body {
  border-bottom-color: #bce8f1;
}
.panel-warning {
  border-color: #faebcc;
}
.panel-warning > .panel-heading {
  color: #8a6d3b;
  background-color: #fcf8e3;
  border-color: #faebcc;
}
.panel-warning > .panel-heading + .panel-collapse > .panel-body {
  border-top-color: #faebcc;
}
.panel-warning > .panel-heading .badge {
  color: #fcf8e3;
  background-color: #8a6d3b;
}
.panel-warning > .panel-footer + .panel-collapse > .panel-body {
  border-bottom-color: #faebcc;
}
.panel-danger {
  border-color: #ebccd1;
}
.panel-danger > .panel-heading {
  color: #a94442;
  background-color: #f2dede;
  border-color: #ebccd1;
}
.panel-danger > .panel-heading + .panel-collapse > .panel-body {
  border-top-color: #ebccd1;
}
.panel-danger > .panel-heading .badge {
  color: #f2dede;
  background-color: #a94442;
}
.panel-danger > .panel-footer + .panel-collapse > .panel-body {
  border-bottom-color: #ebccd1;
}
.embed-responsive {
  position: relative;
  display: block;
  height: 0;
  padding: 0;
  overflow: hidden;
}
.embed-responsive .embed-responsive-item,
.embed-responsive iframe,
.embed-responsive embed,
.embed-responsive object,
.embed-responsive video {
  position: absolute;
  top: 0;
  left: 0;
  bottom: 0;
  height: 100%;
  width: 100%;
  border: 0;
}
.embed-responsive-16by9 {
  padding-bottom: 56.25%;
}
.embed-responsive-4by3 {
  padding-bottom: 75%;
}
.well {
  min-height: 20px;
  padding: 19px;
  margin-bottom: 20px;
  background-color: #f5f5f5;
  border: 1px solid #e3e3e3;
  border-radius: 2px;
  -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.05);
  box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.05);
}
.well blockquote {
  border-color: #ddd;
  border-color: rgba(0, 0, 0, 0.15);
}
.well-lg {
  padding: 24px;
  border-radius: 3px;
}
.well-sm {
  padding: 9px;
  border-radius: 1px;
}
.close {
  float: right;
  font-size: 19.5px;
  font-weight: bold;
  line-height: 1;
  color: #000;
  text-shadow: 0 1px 0 #fff;
  opacity: 0.2;
  filter: alpha(opacity=20);
}
.close:hover,
.close:focus {
  color: #000;
  text-decoration: none;
  cursor: pointer;
  opacity: 0.5;
  filter: alpha(opacity=50);
}
button.close {
  padding: 0;
  cursor: pointer;
  background: transparent;
  border: 0;
  -webkit-appearance: none;
}
.modal-open {
  overflow: hidden;
}
.modal {
  display: none;
  overflow: hidden;
  position: fixed;
  top: 0;
  right: 0;
  bottom: 0;
  left: 0;
  z-index: 1050;
  -webkit-overflow-scrolling: touch;
  outline: 0;
}
.modal.fade .modal-dialog {
  -webkit-transform: translate(0, -25%);
  -ms-transform: translate(0, -25%);
  -o-transform: translate(0, -25%);
  transform: translate(0, -25%);
  -webkit-transition: -webkit-transform 0.3s ease-out;
  -moz-transition: -moz-transform 0.3s ease-out;
  -o-transition: -o-transform 0.3s ease-out;
  transition: transform 0.3s ease-out;
}
.modal.in .modal-dialog {
  -webkit-transform: translate(0, 0);
  -ms-transform: translate(0, 0);
  -o-transform: translate(0, 0);
  transform: translate(0, 0);
}
.modal-open .modal {
  overflow-x: hidden;
  overflow-y: auto;
}
.modal-dialog {
  position: relative;
  width: auto;
  margin: 10px;
}
.modal-content {
  position: relative;
  background-color: #fff;
  border: 1px solid #999;
  border: 1px solid rgba(0, 0, 0, 0.2);
  border-radius: 3px;
  -webkit-box-shadow: 0 3px 9px rgba(0, 0, 0, 0.5);
  box-shadow: 0 3px 9px rgba(0, 0, 0, 0.5);
  background-clip: padding-box;
  outline: 0;
}
.modal-backdrop {
  position: fixed;
  top: 0;
  right: 0;
  bottom: 0;
  left: 0;
  z-index: 1040;
  background-color: #000;
}
.modal-backdrop.fade {
  opacity: 0;
  filter: alpha(opacity=0);
}
.modal-backdrop.in {
  opacity: 0.5;
  filter: alpha(opacity=50);
}
.modal-header {
  padding: 15px;
  border-bottom: 1px solid #e5e5e5;
}
.modal-header .close {
  margin-top: -2px;
}
.modal-title {
  margin: 0;
  line-height: 1.42857143;
}
.modal-body {
  position: relative;
  padding: 15px;
}
.modal-footer {
  padding: 15px;
  text-align: right;
  border-top: 1px solid #e5e5e5;
}
.modal-footer .btn + .btn {
  margin-left: 5px;
  margin-bottom: 0;
}
.modal-footer .btn-group .btn + .btn {
  margin-left: -1px;
}
.modal-footer .btn-block + .btn-block {
  margin-left: 0;
}
.modal-scrollbar-measure {
  position: absolute;
  top: -9999px;
  width: 50px;
  height: 50px;
  overflow: scroll;
}
@media (min-width: 768px) {
  .modal-dialog {
    width: 600px;
    margin: 30px auto;
  }
  .modal-content {
    -webkit-box-shadow: 0 5px 15px rgba(0, 0, 0, 0.5);
    box-shadow: 0 5px 15px rgba(0, 0, 0, 0.5);
  }
  .modal-sm {
    width: 300px;
  }
}
@media (min-width: 992px) {
  .modal-lg {
    width: 900px;
  }
}
.tooltip {
  position: absolute;
  z-index: 1070;
  display: block;
  font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
  font-style: normal;
  font-weight: normal;
  letter-spacing: normal;
  line-break: auto;
  line-height: 1.42857143;
  text-align: left;
  text-align: start;
  text-decoration: none;
  text-shadow: none;
  text-transform: none;
  white-space: normal;
  word-break: normal;
  word-spacing: normal;
  word-wrap: normal;
  font-size: 12px;
  opacity: 0;
  filter: alpha(opacity=0);
}
.tooltip.in {
  opacity: 0.9;
  filter: alpha(opacity=90);
}
.tooltip.top {
  margin-top: -3px;
  padding: 5px 0;
}
.tooltip.right {
  margin-left: 3px;
  padding: 0 5px;
}
.tooltip.bottom {
  margin-top: 3px;
  padding: 5px 0;
}
.tooltip.left {
  margin-left: -3px;
  padding: 0 5px;
}
.tooltip-inner {
  max-width: 200px;
  padding: 3px 8px;
  color: #fff;
  text-align: center;
  background-color: #000;
  border-radius: 2px;
}
.tooltip-arrow {
  position: absolute;
  width: 0;
  height: 0;
  border-color: transparent;
  border-style: solid;
}
.tooltip.top .tooltip-arrow {
  bottom: 0;
  left: 50%;
  margin-left: -5px;
  border-width: 5px 5px 0;
  border-top-color: #000;
}
.tooltip.top-left .tooltip-arrow {
  bottom: 0;
  right: 5px;
  margin-bottom: -5px;
  border-width: 5px 5px 0;
  border-top-color: #000;
}
.tooltip.top-right .tooltip-arrow {
  bottom: 0;
  left: 5px;
  margin-bottom: -5px;
  border-width: 5px 5px 0;
  border-top-color: #000;
}
.tooltip.right .tooltip-arrow {
  top: 50%;
  left: 0;
  margin-top: -5px;
  border-width: 5px 5px 5px 0;
  border-right-color: #000;
}
.tooltip.left .tooltip-arrow {
  top: 50%;
  right: 0;
  margin-top: -5px;
  border-width: 5px 0 5px 5px;
  border-left-color: #000;
}
.tooltip.bottom .tooltip-arrow {
  top: 0;
  left: 50%;
  margin-left: -5px;
  border-width: 0 5px 5px;
  border-bottom-color: #000;
}
.tooltip.bottom-left .tooltip-arrow {
  top: 0;
  right: 5px;
  margin-top: -5px;
  border-width: 0 5px 5px;
  border-bottom-color: #000;
}
.tooltip.bottom-right .tooltip-arrow {
  top: 0;
  left: 5px;
  margin-top: -5px;
  border-width: 0 5px 5px;
  border-bottom-color: #000;
}
.popover {
  position: absolute;
  top: 0;
  left: 0;
  z-index: 1060;
  display: none;
  max-width: 276px;
  padding: 1px;
  font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
  font-style: normal;
  font-weight: normal;
  letter-spacing: normal;
  line-break: auto;
  line-height: 1.42857143;
  text-align: left;
  text-align: start;
  text-decoration: none;
  text-shadow: none;
  text-transform: none;
  white-space: normal;
  word-break: normal;
  word-spacing: normal;
  word-wrap: normal;
  font-size: 13px;
  background-color: #fff;
  background-clip: padding-box;
  border: 1px solid #ccc;
  border: 1px solid rgba(0, 0, 0, 0.2);
  border-radius: 3px;
  -webkit-box-shadow: 0 5px 10px rgba(0, 0, 0, 0.2);
  box-shadow: 0 5px 10px rgba(0, 0, 0, 0.2);
}
.popover.top {
  margin-top: -10px;
}
.popover.right {
  margin-left: 10px;
}
.popover.bottom {
  margin-top: 10px;
}
.popover.left {
  margin-left: -10px;
}
.popover-title {
  margin: 0;
  padding: 8px 14px;
  font-size: 13px;
  background-color: #f7f7f7;
  border-bottom: 1px solid #ebebeb;
  border-radius: 2px 2px 0 0;
}
.popover-content {
  padding: 9px 14px;
}
.popover > .arrow,
.popover > .arrow:after {
  position: absolute;
  display: block;
  width: 0;
  height: 0;
  border-color: transparent;
  border-style: solid;
}
.popover > .arrow {
  border-width: 11px;
}
.popover > .arrow:after {
  border-width: 10px;
  content: "";
}
.popover.top > .arrow {
  left: 50%;
  margin-left: -11px;
  border-bottom-width: 0;
  border-top-color: #999999;
  border-top-color: rgba(0, 0, 0, 0.25);
  bottom: -11px;
}
.popover.top > .arrow:after {
  content: " ";
  bottom: 1px;
  margin-left: -10px;
  border-bottom-width: 0;
  border-top-color: #fff;
}
.popover.right > .arrow {
  top: 50%;
  left: -11px;
  margin-top: -11px;
  border-left-width: 0;
  border-right-color: #999999;
  border-right-color: rgba(0, 0, 0, 0.25);
}
.popover.right > .arrow:after {
  content: " ";
  left: 1px;
  bottom: -10px;
  border-left-width: 0;
  border-right-color: #fff;
}
.popover.bottom > .arrow {
  left: 50%;
  margin-left: -11px;
  border-top-width: 0;
  border-bottom-color: #999999;
  border-bottom-color: rgba(0, 0, 0, 0.25);
  top: -11px;
}
.popover.bottom > .arrow:after {
  content: " ";
  top: 1px;
  margin-left: -10px;
  border-top-width: 0;
  border-bottom-color: #fff;
}
.popover.left > .arrow {
  top: 50%;
  right: -11px;
  margin-top: -11px;
  border-right-width: 0;
  border-left-color: #999999;
  border-left-color: rgba(0, 0, 0, 0.25);
}
.popover.left > .arrow:after {
  content: " ";
  right: 1px;
  border-right-width: 0;
  border-left-color: #fff;
  bottom: -10px;
}
.carousel {
  position: relative;
}
.carousel-inner {
  position: relative;
  overflow: hidden;
  width: 100%;
}
.carousel-inner > .item {
  display: none;
  position: relative;
  -webkit-transition: 0.6s ease-in-out left;
  -o-transition: 0.6s ease-in-out left;
  transition: 0.6s ease-in-out left;
}
.carousel-inner > .item > img,
.carousel-inner > .item > a > img {
  line-height: 1;
}
@media all and (transform-3d), (-webkit-transform-3d) {
  .carousel-inner > .item {
    -webkit-transition: -webkit-transform 0.6s ease-in-out;
    -moz-transition: -moz-transform 0.6s ease-in-out;
    -o-transition: -o-transform 0.6s ease-in-out;
    transition: transform 0.6s ease-in-out;
    -webkit-backface-visibility: hidden;
    -moz-backface-visibility: hidden;
    backface-visibility: hidden;
    -webkit-perspective: 1000px;
    -moz-perspective: 1000px;
    perspective: 1000px;
  }
  .carousel-inner > .item.next,
  .carousel-inner > .item.active.right {
    -webkit-transform: translate3d(100%, 0, 0);
    transform: translate3d(100%, 0, 0);
    left: 0;
  }
  .carousel-inner > .item.prev,
  .carousel-inner > .item.active.left {
    -webkit-transform: translate3d(-100%, 0, 0);
    transform: translate3d(-100%, 0, 0);
    left: 0;
  }
  .carousel-inner > .item.next.left,
  .carousel-inner > .item.prev.right,
  .carousel-inner > .item.active {
    -webkit-transform: translate3d(0, 0, 0);
    transform: translate3d(0, 0, 0);
    left: 0;
  }
}
.carousel-inner > .active,
.carousel-inner > .next,
.carousel-inner > .prev {
  display: block;
}
.carousel-inner > .active {
  left: 0;
}
.carousel-inner > .next,
.carousel-inner > .prev {
  position: absolute;
  top: 0;
  width: 100%;
}
.carousel-inner > .next {
  left: 100%;
}
.carousel-inner > .prev {
  left: -100%;
}
.carousel-inner > .next.left,
.carousel-inner > .prev.right {
  left: 0;
}
.carousel-inner > .active.left {
  left: -100%;
}
.carousel-inner > .active.right {
  left: 100%;
}
.carousel-control {
  position: absolute;
  top: 0;
  left: 0;
  bottom: 0;
  width: 15%;
  opacity: 0.5;
  filter: alpha(opacity=50);
  font-size: 20px;
  color: #fff;
  text-align: center;
  text-shadow: 0 1px 2px rgba(0, 0, 0, 0.6);
  background-color: rgba(0, 0, 0, 0);
}
.carousel-control.left {
  background-image: -webkit-linear-gradient(left, rgba(0, 0, 0, 0.5) 0%, rgba(0, 0, 0, 0.0001) 100%);
  background-image: -o-linear-gradient(left, rgba(0, 0, 0, 0.5) 0%, rgba(0, 0, 0, 0.0001) 100%);
  background-image: linear-gradient(to right, rgba(0, 0, 0, 0.5) 0%, rgba(0, 0, 0, 0.0001) 100%);
  background-repeat: repeat-x;
  filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#80000000', endColorstr='#00000000', GradientType=1);
}
.carousel-control.right {
  left: auto;
  right: 0;
  background-image: -webkit-linear-gradient(left, rgba(0, 0, 0, 0.0001) 0%, rgba(0, 0, 0, 0.5) 100%);
  background-image: -o-linear-gradient(left, rgba(0, 0, 0, 0.0001) 0%, rgba(0, 0, 0, 0.5) 100%);
  background-image: linear-gradient(to right, rgba(0, 0, 0, 0.0001) 0%, rgba(0, 0, 0, 0.5) 100%);
  background-repeat: repeat-x;
  filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#00000000', endColorstr='#80000000', GradientType=1);
}
.carousel-control:hover,
.carousel-control:focus {
  outline: 0;
  color: #fff;
  text-decoration: none;
  opacity: 0.9;
  filter: alpha(opacity=90);
}
.carousel-control .icon-prev,
.carousel-control .icon-next,
.carousel-control .glyphicon-chevron-left,
.carousel-control .glyphicon-chevron-right {
  position: absolute;
  top: 50%;
  margin-top: -10px;
  z-index: 5;
  display: inline-block;
}
.carousel-control .icon-prev,
.carousel-control .glyphicon-chevron-left {
  left: 50%;
  margin-left: -10px;
}
.carousel-control .icon-next,
.carousel-control .glyphicon-chevron-right {
  right: 50%;
  margin-right: -10px;
}
.carousel-control .icon-prev,
.carousel-control .icon-next {
  width: 20px;
  height: 20px;
  line-height: 1;
  font-family: serif;
}
.carousel-control .icon-prev:before {
  content: '\2039';
}
.carousel-control .icon-next:before {
  content: '\203a';
}
.carousel-indicators {
  position: absolute;
  bottom: 10px;
  left: 50%;
  z-index: 15;
  width: 60%;
  margin-left: -30%;
  padding-left: 0;
  list-style: none;
  text-align: center;
}
.carousel-indicators li {
  display: inline-block;
  width: 10px;
  height: 10px;
  margin: 1px;
  text-indent: -999px;
  border: 1px solid #fff;
  border-radius: 10px;
  cursor: pointer;
  background-color: #000 \9;
  background-color: rgba(0, 0, 0, 0);
}
.carousel-indicators .active {
  margin: 0;
  width: 12px;
  height: 12px;
  background-color: #fff;
}
.carousel-caption {
  position: absolute;
  left: 15%;
  right: 15%;
  bottom: 20px;
  z-index: 10;
  padding-top: 20px;
  padding-bottom: 20px;
  color: #fff;
  text-align: center;
  text-shadow: 0 1px 2px rgba(0, 0, 0, 0.6);
}
.carousel-caption .btn {
  text-shadow: none;
}
@media screen and (min-width: 768px) {
  .carousel-control .glyphicon-chevron-left,
  .carousel-control .glyphicon-chevron-right,
  .carousel-control .icon-prev,
  .carousel-control .icon-next {
    width: 30px;
    height: 30px;
    margin-top: -10px;
    font-size: 30px;
  }
  .carousel-control .glyphicon-chevron-left,
  .carousel-control .icon-prev {
    margin-left: -10px;
  }
  .carousel-control .glyphicon-chevron-right,
  .carousel-control .icon-next {
    margin-right: -10px;
  }
  .carousel-caption {
    left: 20%;
    right: 20%;
    padding-bottom: 30px;
  }
  .carousel-indicators {
    bottom: 20px;
  }
}
.clearfix:before,
.clearfix:after,
.dl-horizontal dd:before,
.dl-horizontal dd:after,
.container:before,
.container:after,
.container-fluid:before,
.container-fluid:after,
.row:before,
.row:after,
.form-horizontal .form-group:before,
.form-horizontal .form-group:after,
.btn-toolbar:before,
.btn-toolbar:after,
.btn-group-vertical > .btn-group:before,
.btn-group-vertical > .btn-group:after,
.nav:before,
.nav:after,
.navbar:before,
.navbar:after,
.navbar-header:before,
.navbar-header:after,
.navbar-collapse:before,
.navbar-collapse:after,
.pager:before,
.pager:after,
.panel-body:before,
.panel-body:after,
.modal-header:before,
.modal-header:after,
.modal-footer:before,
.modal-footer:after,
.item_buttons:before,
.item_buttons:after {
  content: " ";
  display: table;
}
.clearfix:after,
.dl-horizontal dd:after,
.container:after,
.container-fluid:after,
.row:after,
.form-horizontal .form-group:after,
.btn-toolbar:after,
.btn-group-vertical > .btn-group:after,
.nav:after,
.navbar:after,
.navbar-header:after,
.navbar-collapse:after,
.pager:after,
.panel-body:after,
.modal-header:after,
.modal-footer:after,
.item_buttons:after {
  clear: both;
}
.center-block {
  display: block;
  margin-left: auto;
  margin-right: auto;
}
.pull-right {
  float: right !important;
}
.pull-left {
  float: left !important;
}
.hide {
  display: none !important;
}
.show {
  display: block !important;
}
.invisible {
  visibility: hidden;
}
.text-hide {
  font: 0/0 a;
  color: transparent;
  text-shadow: none;
  background-color: transparent;
  border: 0;
}
.hidden {
  display: none !important;
}
.affix {
  position: fixed;
}
@-ms-viewport {
  width: device-width;
}
.visible-xs,
.visible-sm,
.visible-md,
.visible-lg {
  display: none !important;
}
.visible-xs-block,
.visible-xs-inline,
.visible-xs-inline-block,
.visible-sm-block,
.visible-sm-inline,
.visible-sm-inline-block,
.visible-md-block,
.visible-md-inline,
.visible-md-inline-block,
.visible-lg-block,
.visible-lg-inline,
.visible-lg-inline-block {
  display: none !important;
}
@media (max-width: 767px) {
  .visible-xs {
    display: block !important;
  }
  table.visible-xs {
    display: table !important;
  }
  tr.visible-xs {
    display: table-row !important;
  }
  th.visible-xs,
  td.visible-xs {
    display: table-cell !important;
  }
}
@media (max-width: 767px) {
  .visible-xs-block {
    display: block !important;
  }
}
@media (max-width: 767px) {
  .visible-xs-inline {
    display: inline !important;
  }
}
@media (max-width: 767px) {
  .visible-xs-inline-block {
    display: inline-block !important;
  }
}
@media (min-width: 768px) and (max-width: 991px) {
  .visible-sm {
    display: block !important;
  }
  table.visible-sm {
    display: table !important;
  }
  tr.visible-sm {
    display: table-row !important;
  }
  th.visible-sm,
  td.visible-sm {
    display: table-cell !important;
  }
}
@media (min-width: 768px) and (max-width: 991px) {
  .visible-sm-block {
    display: block !important;
  }
}
@media (min-width: 768px) and (max-width: 991px) {
  .visible-sm-inline {
    display: inline !important;
  }
}
@media (min-width: 768px) and (max-width: 991px) {
  .visible-sm-inline-block {
    display: inline-block !important;
  }
}
@media (min-width: 992px) and (max-width: 1199px) {
  .visible-md {
    display: block !important;
  }
  table.visible-md {
    display: table !important;
  }
  tr.visible-md {
    display: table-row !important;
  }
  th.visible-md,
  td.visible-md {
    display: table-cell !important;
  }
}
@media (min-width: 992px) and (max-width: 1199px) {
  .visible-md-block {
    display: block !important;
  }
}
@media (min-width: 992px) and (max-width: 1199px) {
  .visible-md-inline {
    display: inline !important;
  }
}
@media (min-width: 992px) and (max-width: 1199px) {
  .visible-md-inline-block {
    display: inline-block !important;
  }
}
@media (min-width: 1200px) {
  .visible-lg {
    display: block !important;
  }
  table.visible-lg {
    display: table !important;
  }
  tr.visible-lg {
    display: table-row !important;
  }
  th.visible-lg,
  td.visible-lg {
    display: table-cell !important;
  }
}
@media (min-width: 1200px) {
  .visible-lg-block {
    display: block !important;
  }
}
@media (min-width: 1200px) {
  .visible-lg-inline {
    display: inline !important;
  }
}
@media (min-width: 1200px) {
  .visible-lg-inline-block {
    display: inline-block !important;
  }
}
@media (max-width: 767px) {
  .hidden-xs {
    display: none !important;
  }
}
@media (min-width: 768px) and (max-width: 991px) {
  .hidden-sm {
    display: none !important;
  }
}
@media (min-width: 992px) and (max-width: 1199px) {
  .hidden-md {
    display: none !important;
  }
}
@media (min-width: 1200px) {
  .hidden-lg {
    display: none !important;
  }
}
.visible-print {
  display: none !important;
}
@media print {
  .visible-print {
    display: block !important;
  }
  table.visible-print {
    display: table !important;
  }
  tr.visible-print {
    display: table-row !important;
  }
  th.visible-print,
  td.visible-print {
    display: table-cell !important;
  }
}
.visible-print-block {
  display: none !important;
}
@media print {
  .visible-print-block {
    display: block !important;
  }
}
.visible-print-inline {
  display: none !important;
}
@media print {
  .visible-print-inline {
    display: inline !important;
  }
}
.visible-print-inline-block {
  display: none !important;
}
@media print {
  .visible-print-inline-block {
    display: inline-block !important;
  }
}
@media print {
  .hidden-print {
    display: none !important;
  }
}
/*!
*
* Font Awesome
*
*/
/*!
 *  Font Awesome 4.7.0 by @davegandy - http://fontawesome.io - @fontawesome
 *  License - http://fontawesome.io/license (Font: SIL OFL 1.1, CSS: MIT License)
 */
/* FONT PATH
 * -------------------------- */
@font-face {
  font-family: 'FontAwesome';
  src: url('../components/font-awesome/fonts/fontawesome-webfont.eot?v=4.7.0');
  src: url('../components/font-awesome/fonts/fontawesome-webfont.eot?#iefix&v=4.7.0') format('embedded-opentype'), url('../components/font-awesome/fonts/fontawesome-webfont.woff2?v=4.7.0') format('woff2'), url('../components/font-awesome/fonts/fontawesome-webfont.woff?v=4.7.0') format('woff'), url('../components/font-awesome/fonts/fontawesome-webfont.ttf?v=4.7.0') format('truetype'), url('../components/font-awesome/fonts/fontawesome-webfont.svg?v=4.7.0#fontawesomeregular') format('svg');
  font-weight: normal;
  font-style: normal;
}
.fa {
  display: inline-block;
  font: normal normal normal 14px/1 FontAwesome;
  font-size: inherit;
  text-rendering: auto;
  -webkit-font-smoothing: antialiased;
  -moz-osx-font-smoothing: grayscale;
}
/* makes the font 33% larger relative to the icon container */
.fa-lg {
  font-size: 1.33333333em;
  line-height: 0.75em;
  vertical-align: -15%;
}
.fa-2x {
  font-size: 2em;
}
.fa-3x {
  font-size: 3em;
}
.fa-4x {
  font-size: 4em;
}
.fa-5x {
  font-size: 5em;
}
.fa-fw {
  width: 1.28571429em;
  text-align: center;
}
.fa-ul {
  padding-left: 0;
  margin-left: 2.14285714em;
  list-style-type: none;
}
.fa-ul > li {
  position: relative;
}
.fa-li {
  position: absolute;
  left: -2.14285714em;
  width: 2.14285714em;
  top: 0.14285714em;
  text-align: center;
}
.fa-li.fa-lg {
  left: -1.85714286em;
}
.fa-border {
  padding: .2em .25em .15em;
  border: solid 0.08em #eee;
  border-radius: .1em;
}
.fa-pull-left {
  float: left;
}
.fa-pull-right {
  float: right;
}
.fa.fa-pull-left {
  margin-right: .3em;
}
.fa.fa-pull-right {
  margin-left: .3em;
}
/* Deprecated as of 4.4.0 */
.pull-right {
  float: right;
}
.pull-left {
  float: left;
}
.fa.pull-left {
  margin-right: .3em;
}
.fa.pull-right {
  margin-left: .3em;
}
.fa-spin {
  -webkit-animation: fa-spin 2s infinite linear;
  animation: fa-spin 2s infinite linear;
}
.fa-pulse {
  -webkit-animation: fa-spin 1s infinite steps(8);
  animation: fa-spin 1s infinite steps(8);
}
@-webkit-keyframes fa-spin {
  0% {
    -webkit-transform: rotate(0deg);
    transform: rotate(0deg);
  }
  100% {
    -webkit-transform: rotate(359deg);
    transform: rotate(359deg);
  }
}
@keyframes fa-spin {
  0% {
    -webkit-transform: rotate(0deg);
    transform: rotate(0deg);
  }
  100% {
    -webkit-transform: rotate(359deg);
    transform: rotate(359deg);
  }
}
.fa-rotate-90 {
  -ms-filter: "progid:DXImageTransform.Microsoft.BasicImage(rotation=1)";
  -webkit-transform: rotate(90deg);
  -ms-transform: rotate(90deg);
  transform: rotate(90deg);
}
.fa-rotate-180 {
  -ms-filter: "progid:DXImageTransform.Microsoft.BasicImage(rotation=2)";
  -webkit-transform: rotate(180deg);
  -ms-transform: rotate(180deg);
  transform: rotate(180deg);
}
.fa-rotate-270 {
  -ms-filter: "progid:DXImageTransform.Microsoft.BasicImage(rotation=3)";
  -webkit-transform: rotate(270deg);
  -ms-transform: rotate(270deg);
  transform: rotate(270deg);
}
.fa-flip-horizontal {
  -ms-filter: "progid:DXImageTransform.Microsoft.BasicImage(rotation=0, mirror=1)";
  -webkit-transform: scale(-1, 1);
  -ms-transform: scale(-1, 1);
  transform: scale(-1, 1);
}
.fa-flip-vertical {
  -ms-filter: "progid:DXImageTransform.Microsoft.BasicImage(rotation=2, mirror=1)";
  -webkit-transform: scale(1, -1);
  -ms-transform: scale(1, -1);
  transform: scale(1, -1);
}
:root .fa-rotate-90,
:root .fa-rotate-180,
:root .fa-rotate-270,
:root .fa-flip-horizontal,
:root .fa-flip-vertical {
  filter: none;
}
.fa-stack {
  position: relative;
  display: inline-block;
  width: 2em;
  height: 2em;
  line-height: 2em;
  vertical-align: middle;
}
.fa-stack-1x,
.fa-stack-2x {
  position: absolute;
  left: 0;
  width: 100%;
  text-align: center;
}
.fa-stack-1x {
  line-height: inherit;
}
.fa-stack-2x {
  font-size: 2em;
}
.fa-inverse {
  color: #fff;
}
/* Font Awesome uses the Unicode Private Use Area (PUA) to ensure screen
   readers do not read off random characters that represent icons */
.fa-glass:before {
  content: "\f000";
}
.fa-music:before {
  content: "\f001";
}
.fa-search:before {
  content: "\f002";
}
.fa-envelope-o:before {
  content: "\f003";
}
.fa-heart:before {
  content: "\f004";
}
.fa-star:before {
  content: "\f005";
}
.fa-star-o:before {
  content: "\f006";
}
.fa-user:before {
  content: "\f007";
}
.fa-film:before {
  content: "\f008";
}
.fa-th-large:before {
  content: "\f009";
}
.fa-th:before {
  content: "\f00a";
}
.fa-th-list:before {
  content: "\f00b";
}
.fa-check:before {
  content: "\f00c";
}
.fa-remove:before,
.fa-close:before,
.fa-times:before {
  content: "\f00d";
}
.fa-search-plus:before {
  content: "\f00e";
}
.fa-search-minus:before {
  content: "\f010";
}
.fa-power-off:before {
  content: "\f011";
}
.fa-signal:before {
  content: "\f012";
}
.fa-gear:before,
.fa-cog:before {
  content: "\f013";
}
.fa-trash-o:before {
  content: "\f014";
}
.fa-home:before {
  content: "\f015";
}
.fa-file-o:before {
  content: "\f016";
}
.fa-clock-o:before {
  content: "\f017";
}
.fa-road:before {
  content: "\f018";
}
.fa-download:before {
  content: "\f019";
}
.fa-arrow-circle-o-down:before {
  content: "\f01a";
}
.fa-arrow-circle-o-up:before {
  content: "\f01b";
}
.fa-inbox:before {
  content: "\f01c";
}
.fa-play-circle-o:before {
  content: "\f01d";
}
.fa-rotate-right:before,
.fa-repeat:before {
  content: "\f01e";
}
.fa-refresh:before {
  content: "\f021";
}
.fa-list-alt:before {
  content: "\f022";
}
.fa-lock:before {
  content: "\f023";
}
.fa-flag:before {
  content: "\f024";
}
.fa-headphones:before {
  content: "\f025";
}
.fa-volume-off:before {
  content: "\f026";
}
.fa-volume-down:before {
  content: "\f027";
}
.fa-volume-up:before {
  content: "\f028";
}
.fa-qrcode:before {
  content: "\f029";
}
.fa-barcode:before {
  content: "\f02a";
}
.fa-tag:before {
  content: "\f02b";
}
.fa-tags:before {
  content: "\f02c";
}
.fa-book:before {
  content: "\f02d";
}
.fa-bookmark:before {
  content: "\f02e";
}
.fa-print:before {
  content: "\f02f";
}
.fa-camera:before {
  content: "\f030";
}
.fa-font:before {
  content: "\f031";
}
.fa-bold:before {
  content: "\f032";
}
.fa-italic:before {
  content: "\f033";
}
.fa-text-height:before {
  content: "\f034";
}
.fa-text-width:before {
  content: "\f035";
}
.fa-align-left:before {
  content: "\f036";
}
.fa-align-center:before {
  content: "\f037";
}
.fa-align-right:before {
  content: "\f038";
}
.fa-align-justify:before {
  content: "\f039";
}
.fa-list:before {
  content: "\f03a";
}
.fa-dedent:before,
.fa-outdent:before {
  content: "\f03b";
}
.fa-indent:before {
  content: "\f03c";
}
.fa-video-camera:before {
  content: "\f03d";
}
.fa-photo:before,
.fa-image:before,
.fa-picture-o:before {
  content: "\f03e";
}
.fa-pencil:before {
  content: "\f040";
}
.fa-map-marker:before {
  content: "\f041";
}
.fa-adjust:before {
  content: "\f042";
}
.fa-tint:before {
  content: "\f043";
}
.fa-edit:before,
.fa-pencil-square-o:before {
  content: "\f044";
}
.fa-share-square-o:before {
  content: "\f045";
}
.fa-check-square-o:before {
  content: "\f046";
}
.fa-arrows:before {
  content: "\f047";
}
.fa-step-backward:before {
  content: "\f048";
}
.fa-fast-backward:before {
  content: "\f049";
}
.fa-backward:before {
  content: "\f04a";
}
.fa-play:before {
  content: "\f04b";
}
.fa-pause:before {
  content: "\f04c";
}
.fa-stop:before {
  content: "\f04d";
}
.fa-forward:before {
  content: "\f04e";
}
.fa-fast-forward:before {
  content: "\f050";
}
.fa-step-forward:before {
  content: "\f051";
}
.fa-eject:before {
  content: "\f052";
}
.fa-chevron-left:before {
  content: "\f053";
}
.fa-chevron-right:before {
  content: "\f054";
}
.fa-plus-circle:before {
  content: "\f055";
}
.fa-minus-circle:before {
  content: "\f056";
}
.fa-times-circle:before {
  content: "\f057";
}
.fa-check-circle:before {
  content: "\f058";
}
.fa-question-circle:before {
  content: "\f059";
}
.fa-info-circle:before {
  content: "\f05a";
}
.fa-crosshairs:before {
  content: "\f05b";
}
.fa-times-circle-o:before {
  content: "\f05c";
}
.fa-check-circle-o:before {
  content: "\f05d";
}
.fa-ban:before {
  content: "\f05e";
}
.fa-arrow-left:before {
  content: "\f060";
}
.fa-arrow-right:before {
  content: "\f061";
}
.fa-arrow-up:before {
  content: "\f062";
}
.fa-arrow-down:before {
  content: "\f063";
}
.fa-mail-forward:before,
.fa-share:before {
  content: "\f064";
}
.fa-expand:before {
  content: "\f065";
}
.fa-compress:before {
  content: "\f066";
}
.fa-plus:before {
  content: "\f067";
}
.fa-minus:before {
  content: "\f068";
}
.fa-asterisk:before {
  content: "\f069";
}
.fa-exclamation-circle:before {
  content: "\f06a";
}
.fa-gift:before {
  content: "\f06b";
}
.fa-leaf:before {
  content: "\f06c";
}
.fa-fire:before {
  content: "\f06d";
}
.fa-eye:before {
  content: "\f06e";
}
.fa-eye-slash:before {
  content: "\f070";
}
.fa-warning:before,
.fa-exclamation-triangle:before {
  content: "\f071";
}
.fa-plane:before {
  content: "\f072";
}
.fa-calendar:before {
  content: "\f073";
}
.fa-random:before {
  content: "\f074";
}
.fa-comment:before {
  content: "\f075";
}
.fa-magnet:before {
  content: "\f076";
}
.fa-chevron-up:before {
  content: "\f077";
}
.fa-chevron-down:before {
  content: "\f078";
}
.fa-retweet:before {
  content: "\f079";
}
.fa-shopping-cart:before {
  content: "\f07a";
}
.fa-folder:before {
  content: "\f07b";
}
.fa-folder-open:before {
  content: "\f07c";
}
.fa-arrows-v:before {
  content: "\f07d";
}
.fa-arrows-h:before {
  content: "\f07e";
}
.fa-bar-chart-o:before,
.fa-bar-chart:before {
  content: "\f080";
}
.fa-twitter-square:before {
  content: "\f081";
}
.fa-facebook-square:before {
  content: "\f082";
}
.fa-camera-retro:before {
  content: "\f083";
}
.fa-key:before {
  content: "\f084";
}
.fa-gears:before,
.fa-cogs:before {
  content: "\f085";
}
.fa-comments:before {
  content: "\f086";
}
.fa-thumbs-o-up:before {
  content: "\f087";
}
.fa-thumbs-o-down:before {
  content: "\f088";
}
.fa-star-half:before {
  content: "\f089";
}
.fa-heart-o:before {
  content: "\f08a";
}
.fa-sign-out:before {
  content: "\f08b";
}
.fa-linkedin-square:before {
  content: "\f08c";
}
.fa-thumb-tack:before {
  content: "\f08d";
}
.fa-external-link:before {
  content: "\f08e";
}
.fa-sign-in:before {
  content: "\f090";
}
.fa-trophy:before {
  content: "\f091";
}
.fa-github-square:before {
  content: "\f092";
}
.fa-upload:before {
  content: "\f093";
}
.fa-lemon-o:before {
  content: "\f094";
}
.fa-phone:before {
  content: "\f095";
}
.fa-square-o:before {
  content: "\f096";
}
.fa-bookmark-o:before {
  content: "\f097";
}
.fa-phone-square:before {
  content: "\f098";
}
.fa-twitter:before {
  content: "\f099";
}
.fa-facebook-f:before,
.fa-facebook:before {
  content: "\f09a";
}
.fa-github:before {
  content: "\f09b";
}
.fa-unlock:before {
  content: "\f09c";
}
.fa-credit-card:before {
  content: "\f09d";
}
.fa-feed:before,
.fa-rss:before {
  content: "\f09e";
}
.fa-hdd-o:before {
  content: "\f0a0";
}
.fa-bullhorn:before {
  content: "\f0a1";
}
.fa-bell:before {
  content: "\f0f3";
}
.fa-certificate:before {
  content: "\f0a3";
}
.fa-hand-o-right:before {
  content: "\f0a4";
}
.fa-hand-o-left:before {
  content: "\f0a5";
}
.fa-hand-o-up:before {
  content: "\f0a6";
}
.fa-hand-o-down:before {
  content: "\f0a7";
}
.fa-arrow-circle-left:before {
  content: "\f0a8";
}
.fa-arrow-circle-right:before {
  content: "\f0a9";
}
.fa-arrow-circle-up:before {
  content: "\f0aa";
}
.fa-arrow-circle-down:before {
  content: "\f0ab";
}
.fa-globe:before {
  content: "\f0ac";
}
.fa-wrench:before {
  content: "\f0ad";
}
.fa-tasks:before {
  content: "\f0ae";
}
.fa-filter:before {
  content: "\f0b0";
}
.fa-briefcase:before {
  content: "\f0b1";
}
.fa-arrows-alt:before {
  content: "\f0b2";
}
.fa-group:before,
.fa-users:before {
  content: "\f0c0";
}
.fa-chain:before,
.fa-link:before {
  content: "\f0c1";
}
.fa-cloud:before {
  content: "\f0c2";
}
.fa-flask:before {
  content: "\f0c3";
}
.fa-cut:before,
.fa-scissors:before {
  content: "\f0c4";
}
.fa-copy:before,
.fa-files-o:before {
  content: "\f0c5";
}
.fa-paperclip:before {
  content: "\f0c6";
}
.fa-save:before,
.fa-floppy-o:before {
  content: "\f0c7";
}
.fa-square:before {
  content: "\f0c8";
}
.fa-navicon:before,
.fa-reorder:before,
.fa-bars:before {
  content: "\f0c9";
}
.fa-list-ul:before {
  content: "\f0ca";
}
.fa-list-ol:before {
  content: "\f0cb";
}
.fa-strikethrough:before {
  content: "\f0cc";
}
.fa-underline:before {
  content: "\f0cd";
}
.fa-table:before {
  content: "\f0ce";
}
.fa-magic:before {
  content: "\f0d0";
}
.fa-truck:before {
  content: "\f0d1";
}
.fa-pinterest:before {
  content: "\f0d2";
}
.fa-pinterest-square:before {
  content: "\f0d3";
}
.fa-google-plus-square:before {
  content: "\f0d4";
}
.fa-google-plus:before {
  content: "\f0d5";
}
.fa-money:before {
  content: "\f0d6";
}
.fa-caret-down:before {
  content: "\f0d7";
}
.fa-caret-up:before {
  content: "\f0d8";
}
.fa-caret-left:before {
  content: "\f0d9";
}
.fa-caret-right:before {
  content: "\f0da";
}
.fa-columns:before {
  content: "\f0db";
}
.fa-unsorted:before,
.fa-sort:before {
  content: "\f0dc";
}
.fa-sort-down:before,
.fa-sort-desc:before {
  content: "\f0dd";
}
.fa-sort-up:before,
.fa-sort-asc:before {
  content: "\f0de";
}
.fa-envelope:before {
  content: "\f0e0";
}
.fa-linkedin:before {
  content: "\f0e1";
}
.fa-rotate-left:before,
.fa-undo:before {
  content: "\f0e2";
}
.fa-legal:before,
.fa-gavel:before {
  content: "\f0e3";
}
.fa-dashboard:before,
.fa-tachometer:before {
  content: "\f0e4";
}
.fa-comment-o:before {
  content: "\f0e5";
}
.fa-comments-o:before {
  content: "\f0e6";
}
.fa-flash:before,
.fa-bolt:before {
  content: "\f0e7";
}
.fa-sitemap:before {
  content: "\f0e8";
}
.fa-umbrella:before {
  content: "\f0e9";
}
.fa-paste:before,
.fa-clipboard:before {
  content: "\f0ea";
}
.fa-lightbulb-o:before {
  content: "\f0eb";
}
.fa-exchange:before {
  content: "\f0ec";
}
.fa-cloud-download:before {
  content: "\f0ed";
}
.fa-cloud-upload:before {
  content: "\f0ee";
}
.fa-user-md:before {
  content: "\f0f0";
}
.fa-stethoscope:before {
  content: "\f0f1";
}
.fa-suitcase:before {
  content: "\f0f2";
}
.fa-bell-o:before {
  content: "\f0a2";
}
.fa-coffee:before {
  content: "\f0f4";
}
.fa-cutlery:before {
  content: "\f0f5";
}
.fa-file-text-o:before {
  content: "\f0f6";
}
.fa-building-o:before {
  content: "\f0f7";
}
.fa-hospital-o:before {
  content: "\f0f8";
}
.fa-ambulance:before {
  content: "\f0f9";
}
.fa-medkit:before {
  content: "\f0fa";
}
.fa-fighter-jet:before {
  content: "\f0fb";
}
.fa-beer:before {
  content: "\f0fc";
}
.fa-h-square:before {
  content: "\f0fd";
}
.fa-plus-square:before {
  content: "\f0fe";
}
.fa-angle-double-left:before {
  content: "\f100";
}
.fa-angle-double-right:before {
  content: "\f101";
}
.fa-angle-double-up:before {
  content: "\f102";
}
.fa-angle-double-down:before {
  content: "\f103";
}
.fa-angle-left:before {
  content: "\f104";
}
.fa-angle-right:before {
  content: "\f105";
}
.fa-angle-up:before {
  content: "\f106";
}
.fa-angle-down:before {
  content: "\f107";
}
.fa-desktop:before {
  content: "\f108";
}
.fa-laptop:before {
  content: "\f109";
}
.fa-tablet:before {
  content: "\f10a";
}
.fa-mobile-phone:before,
.fa-mobile:before {
  content: "\f10b";
}
.fa-circle-o:before {
  content: "\f10c";
}
.fa-quote-left:before {
  content: "\f10d";
}
.fa-quote-right:before {
  content: "\f10e";
}
.fa-spinner:before {
  content: "\f110";
}
.fa-circle:before {
  content: "\f111";
}
.fa-mail-reply:before,
.fa-reply:before {
  content: "\f112";
}
.fa-github-alt:before {
  content: "\f113";
}
.fa-folder-o:before {
  content: "\f114";
}
.fa-folder-open-o:before {
  content: "\f115";
}
.fa-smile-o:before {
  content: "\f118";
}
.fa-frown-o:before {
  content: "\f119";
}
.fa-meh-o:before {
  content: "\f11a";
}
.fa-gamepad:before {
  content: "\f11b";
}
.fa-keyboard-o:before {
  content: "\f11c";
}
.fa-flag-o:before {
  content: "\f11d";
}
.fa-flag-checkered:before {
  content: "\f11e";
}
.fa-terminal:before {
  content: "\f120";
}
.fa-code:before {
  content: "\f121";
}
.fa-mail-reply-all:before,
.fa-reply-all:before {
  content: "\f122";
}
.fa-star-half-empty:before,
.fa-star-half-full:before,
.fa-star-half-o:before {
  content: "\f123";
}
.fa-location-arrow:before {
  content: "\f124";
}
.fa-crop:before {
  content: "\f125";
}
.fa-code-fork:before {
  content: "\f126";
}
.fa-unlink:before,
.fa-chain-broken:before {
  content: "\f127";
}
.fa-question:before {
  content: "\f128";
}
.fa-info:before {
  content: "\f129";
}
.fa-exclamation:before {
  content: "\f12a";
}
.fa-superscript:before {
  content: "\f12b";
}
.fa-subscript:before {
  content: "\f12c";
}
.fa-eraser:before {
  content: "\f12d";
}
.fa-puzzle-piece:before {
  content: "\f12e";
}
.fa-microphone:before {
  content: "\f130";
}
.fa-microphone-slash:before {
  content: "\f131";
}
.fa-shield:before {
  content: "\f132";
}
.fa-calendar-o:before {
  content: "\f133";
}
.fa-fire-extinguisher:before {
  content: "\f134";
}
.fa-rocket:before {
  content: "\f135";
}
.fa-maxcdn:before {
  content: "\f136";
}
.fa-chevron-circle-left:before {
  content: "\f137";
}
.fa-chevron-circle-right:before {
  content: "\f138";
}
.fa-chevron-circle-up:before {
  content: "\f139";
}
.fa-chevron-circle-down:before {
  content: "\f13a";
}
.fa-html5:before {
  content: "\f13b";
}
.fa-css3:before {
  content: "\f13c";
}
.fa-anchor:before {
  content: "\f13d";
}
.fa-unlock-alt:before {
  content: "\f13e";
}
.fa-bullseye:before {
  content: "\f140";
}
.fa-ellipsis-h:before {
  content: "\f141";
}
.fa-ellipsis-v:before {
  content: "\f142";
}
.fa-rss-square:before {
  content: "\f143";
}
.fa-play-circle:before {
  content: "\f144";
}
.fa-ticket:before {
  content: "\f145";
}
.fa-minus-square:before {
  content: "\f146";
}
.fa-minus-square-o:before {
  content: "\f147";
}
.fa-level-up:before {
  content: "\f148";
}
.fa-level-down:before {
  content: "\f149";
}
.fa-check-square:before {
  content: "\f14a";
}
.fa-pencil-square:before {
  content: "\f14b";
}
.fa-external-link-square:before {
  content: "\f14c";
}
.fa-share-square:before {
  content: "\f14d";
}
.fa-compass:before {
  content: "\f14e";
}
.fa-toggle-down:before,
.fa-caret-square-o-down:before {
  content: "\f150";
}
.fa-toggle-up:before,
.fa-caret-square-o-up:before {
  content: "\f151";
}
.fa-toggle-right:before,
.fa-caret-square-o-right:before {
  content: "\f152";
}
.fa-euro:before,
.fa-eur:before {
  content: "\f153";
}
.fa-gbp:before {
  content: "\f154";
}
.fa-dollar:before,
.fa-usd:before {
  content: "\f155";
}
.fa-rupee:before,
.fa-inr:before {
  content: "\f156";
}
.fa-cny:before,
.fa-rmb:before,
.fa-yen:before,
.fa-jpy:before {
  content: "\f157";
}
.fa-ruble:before,
.fa-rouble:before,
.fa-rub:before {
  content: "\f158";
}
.fa-won:before,
.fa-krw:before {
  content: "\f159";
}
.fa-bitcoin:before,
.fa-btc:before {
  content: "\f15a";
}
.fa-file:before {
  content: "\f15b";
}
.fa-file-text:before {
  content: "\f15c";
}
.fa-sort-alpha-asc:before {
  content: "\f15d";
}
.fa-sort-alpha-desc:before {
  content: "\f15e";
}
.fa-sort-amount-asc:before {
  content: "\f160";
}
.fa-sort-amount-desc:before {
  content: "\f161";
}
.fa-sort-numeric-asc:before {
  content: "\f162";
}
.fa-sort-numeric-desc:before {
  content: "\f163";
}
.fa-thumbs-up:before {
  content: "\f164";
}
.fa-thumbs-down:before {
  content: "\f165";
}
.fa-youtube-square:before {
  content: "\f166";
}
.fa-youtube:before {
  content: "\f167";
}
.fa-xing:before {
  content: "\f168";
}
.fa-xing-square:before {
  content: "\f169";
}
.fa-youtube-play:before {
  content: "\f16a";
}
.fa-dropbox:before {
  content: "\f16b";
}
.fa-stack-overflow:before {
  content: "\f16c";
}
.fa-instagram:before {
  content: "\f16d";
}
.fa-flickr:before {
  content: "\f16e";
}
.fa-adn:before {
  content: "\f170";
}
.fa-bitbucket:before {
  content: "\f171";
}
.fa-bitbucket-square:before {
  content: "\f172";
}
.fa-tumblr:before {
  content: "\f173";
}
.fa-tumblr-square:before {
  content: "\f174";
}
.fa-long-arrow-down:before {
  content: "\f175";
}
.fa-long-arrow-up:before {
  content: "\f176";
}
.fa-long-arrow-left:before {
  content: "\f177";
}
.fa-long-arrow-right:before {
  content: "\f178";
}
.fa-apple:before {
  content: "\f179";
}
.fa-windows:before {
  content: "\f17a";
}
.fa-android:before {
  content: "\f17b";
}
.fa-linux:before {
  content: "\f17c";
}
.fa-dribbble:before {
  content: "\f17d";
}
.fa-skype:before {
  content: "\f17e";
}
.fa-foursquare:before {
  content: "\f180";
}
.fa-trello:before {
  content: "\f181";
}
.fa-female:before {
  content: "\f182";
}
.fa-male:before {
  content: "\f183";
}
.fa-gittip:before,
.fa-gratipay:before {
  content: "\f184";
}
.fa-sun-o:before {
  content: "\f185";
}
.fa-moon-o:before {
  content: "\f186";
}
.fa-archive:before {
  content: "\f187";
}
.fa-bug:before {
  content: "\f188";
}
.fa-vk:before {
  content: "\f189";
}
.fa-weibo:before {
  content: "\f18a";
}
.fa-renren:before {
  content: "\f18b";
}
.fa-pagelines:before {
  content: "\f18c";
}
.fa-stack-exchange:before {
  content: "\f18d";
}
.fa-arrow-circle-o-right:before {
  content: "\f18e";
}
.fa-arrow-circle-o-left:before {
  content: "\f190";
}
.fa-toggle-left:before,
.fa-caret-square-o-left:before {
  content: "\f191";
}
.fa-dot-circle-o:before {
  content: "\f192";
}
.fa-wheelchair:before {
  content: "\f193";
}
.fa-vimeo-square:before {
  content: "\f194";
}
.fa-turkish-lira:before,
.fa-try:before {
  content: "\f195";
}
.fa-plus-square-o:before {
  content: "\f196";
}
.fa-space-shuttle:before {
  content: "\f197";
}
.fa-slack:before {
  content: "\f198";
}
.fa-envelope-square:before {
  content: "\f199";
}
.fa-wordpress:before {
  content: "\f19a";
}
.fa-openid:before {
  content: "\f19b";
}
.fa-institution:before,
.fa-bank:before,
.fa-university:before {
  content: "\f19c";
}
.fa-mortar-board:before,
.fa-graduation-cap:before {
  content: "\f19d";
}
.fa-yahoo:before {
  content: "\f19e";
}
.fa-google:before {
  content: "\f1a0";
}
.fa-reddit:before {
  content: "\f1a1";
}
.fa-reddit-square:before {
  content: "\f1a2";
}
.fa-stumbleupon-circle:before {
  content: "\f1a3";
}
.fa-stumbleupon:before {
  content: "\f1a4";
}
.fa-delicious:before {
  content: "\f1a5";
}
.fa-digg:before {
  content: "\f1a6";
}
.fa-pied-piper-pp:before {
  content: "\f1a7";
}
.fa-pied-piper-alt:before {
  content: "\f1a8";
}
.fa-drupal:before {
  content: "\f1a9";
}
.fa-joomla:before {
  content: "\f1aa";
}
.fa-language:before {
  content: "\f1ab";
}
.fa-fax:before {
  content: "\f1ac";
}
.fa-building:before {
  content: "\f1ad";
}
.fa-child:before {
  content: "\f1ae";
}
.fa-paw:before {
  content: "\f1b0";
}
.fa-spoon:before {
  content: "\f1b1";
}
.fa-cube:before {
  content: "\f1b2";
}
.fa-cubes:before {
  content: "\f1b3";
}
.fa-behance:before {
  content: "\f1b4";
}
.fa-behance-square:before {
  content: "\f1b5";
}
.fa-steam:before {
  content: "\f1b6";
}
.fa-steam-square:before {
  content: "\f1b7";
}
.fa-recycle:before {
  content: "\f1b8";
}
.fa-automobile:before,
.fa-car:before {
  content: "\f1b9";
}
.fa-cab:before,
.fa-taxi:before {
  content: "\f1ba";
}
.fa-tree:before {
  content: "\f1bb";
}
.fa-spotify:before {
  content: "\f1bc";
}
.fa-deviantart:before {
  content: "\f1bd";
}
.fa-soundcloud:before {
  content: "\f1be";
}
.fa-database:before {
  content: "\f1c0";
}
.fa-file-pdf-o:before {
  content: "\f1c1";
}
.fa-file-word-o:before {
  content: "\f1c2";
}
.fa-file-excel-o:before {
  content: "\f1c3";
}
.fa-file-powerpoint-o:before {
  content: "\f1c4";
}
.fa-file-photo-o:before,
.fa-file-picture-o:before,
.fa-file-image-o:before {
  content: "\f1c5";
}
.fa-file-zip-o:before,
.fa-file-archive-o:before {
  content: "\f1c6";
}
.fa-file-sound-o:before,
.fa-file-audio-o:before {
  content: "\f1c7";
}
.fa-file-movie-o:before,
.fa-file-video-o:before {
  content: "\f1c8";
}
.fa-file-code-o:before {
  content: "\f1c9";
}
.fa-vine:before {
  content: "\f1ca";
}
.fa-codepen:before {
  content: "\f1cb";
}
.fa-jsfiddle:before {
  content: "\f1cc";
}
.fa-life-bouy:before,
.fa-life-buoy:before,
.fa-life-saver:before,
.fa-support:before,
.fa-life-ring:before {
  content: "\f1cd";
}
.fa-circle-o-notch:before {
  content: "\f1ce";
}
.fa-ra:before,
.fa-resistance:before,
.fa-rebel:before {
  content: "\f1d0";
}
.fa-ge:before,
.fa-empire:before {
  content: "\f1d1";
}
.fa-git-square:before {
  content: "\f1d2";
}
.fa-git:before {
  content: "\f1d3";
}
.fa-y-combinator-square:before,
.fa-yc-square:before,
.fa-hacker-news:before {
  content: "\f1d4";
}
.fa-tencent-weibo:before {
  content: "\f1d5";
}
.fa-qq:before {
  content: "\f1d6";
}
.fa-wechat:before,
.fa-weixin:before {
  content: "\f1d7";
}
.fa-send:before,
.fa-paper-plane:before {
  content: "\f1d8";
}
.fa-send-o:before,
.fa-paper-plane-o:before {
  content: "\f1d9";
}
.fa-history:before {
  content: "\f1da";
}
.fa-circle-thin:before {
  content: "\f1db";
}
.fa-header:before {
  content: "\f1dc";
}
.fa-paragraph:before {
  content: "\f1dd";
}
.fa-sliders:before {
  content: "\f1de";
}
.fa-share-alt:before {
  content: "\f1e0";
}
.fa-share-alt-square:before {
  content: "\f1e1";
}
.fa-bomb:before {
  content: "\f1e2";
}
.fa-soccer-ball-o:before,
.fa-futbol-o:before {
  content: "\f1e3";
}
.fa-tty:before {
  content: "\f1e4";
}
.fa-binoculars:before {
  content: "\f1e5";
}
.fa-plug:before {
  content: "\f1e6";
}
.fa-slideshare:before {
  content: "\f1e7";
}
.fa-twitch:before {
  content: "\f1e8";
}
.fa-yelp:before {
  content: "\f1e9";
}
.fa-newspaper-o:before {
  content: "\f1ea";
}
.fa-wifi:before {
  content: "\f1eb";
}
.fa-calculator:before {
  content: "\f1ec";
}
.fa-paypal:before {
  content: "\f1ed";
}
.fa-google-wallet:before {
  content: "\f1ee";
}
.fa-cc-visa:before {
  content: "\f1f0";
}
.fa-cc-mastercard:before {
  content: "\f1f1";
}
.fa-cc-discover:before {
  content: "\f1f2";
}
.fa-cc-amex:before {
  content: "\f1f3";
}
.fa-cc-paypal:before {
  content: "\f1f4";
}
.fa-cc-stripe:before {
  content: "\f1f5";
}
.fa-bell-slash:before {
  content: "\f1f6";
}
.fa-bell-slash-o:before {
  content: "\f1f7";
}
.fa-trash:before {
  content: "\f1f8";
}
.fa-copyright:before {
  content: "\f1f9";
}
.fa-at:before {
  content: "\f1fa";
}
.fa-eyedropper:before {
  content: "\f1fb";
}
.fa-paint-brush:before {
  content: "\f1fc";
}
.fa-birthday-cake:before {
  content: "\f1fd";
}
.fa-area-chart:before {
  content: "\f1fe";
}
.fa-pie-chart:before {
  content: "\f200";
}
.fa-line-chart:before {
  content: "\f201";
}
.fa-lastfm:before {
  content: "\f202";
}
.fa-lastfm-square:before {
  content: "\f203";
}
.fa-toggle-off:before {
  content: "\f204";
}
.fa-toggle-on:before {
  content: "\f205";
}
.fa-bicycle:before {
  content: "\f206";
}
.fa-bus:before {
  content: "\f207";
}
.fa-ioxhost:before {
  content: "\f208";
}
.fa-angellist:before {
  content: "\f209";
}
.fa-cc:before {
  content: "\f20a";
}
.fa-shekel:before,
.fa-sheqel:before,
.fa-ils:before {
  content: "\f20b";
}
.fa-meanpath:before {
  content: "\f20c";
}
.fa-buysellads:before {
  content: "\f20d";
}
.fa-connectdevelop:before {
  content: "\f20e";
}
.fa-dashcube:before {
  content: "\f210";
}
.fa-forumbee:before {
  content: "\f211";
}
.fa-leanpub:before {
  content: "\f212";
}
.fa-sellsy:before {
  content: "\f213";
}
.fa-shirtsinbulk:before {
  content: "\f214";
}
.fa-simplybuilt:before {
  content: "\f215";
}
.fa-skyatlas:before {
  content: "\f216";
}
.fa-cart-plus:before {
  content: "\f217";
}
.fa-cart-arrow-down:before {
  content: "\f218";
}
.fa-diamond:before {
  content: "\f219";
}
.fa-ship:before {
  content: "\f21a";
}
.fa-user-secret:before {
  content: "\f21b";
}
.fa-motorcycle:before {
  content: "\f21c";
}
.fa-street-view:before {
  content: "\f21d";
}
.fa-heartbeat:before {
  content: "\f21e";
}
.fa-venus:before {
  content: "\f221";
}
.fa-mars:before {
  content: "\f222";
}
.fa-mercury:before {
  content: "\f223";
}
.fa-intersex:before,
.fa-transgender:before {
  content: "\f224";
}
.fa-transgender-alt:before {
  content: "\f225";
}
.fa-venus-double:before {
  content: "\f226";
}
.fa-mars-double:before {
  content: "\f227";
}
.fa-venus-mars:before {
  content: "\f228";
}
.fa-mars-stroke:before {
  content: "\f229";
}
.fa-mars-stroke-v:before {
  content: "\f22a";
}
.fa-mars-stroke-h:before {
  content: "\f22b";
}
.fa-neuter:before {
  content: "\f22c";
}
.fa-genderless:before {
  content: "\f22d";
}
.fa-facebook-official:before {
  content: "\f230";
}
.fa-pinterest-p:before {
  content: "\f231";
}
.fa-whatsapp:before {
  content: "\f232";
}
.fa-server:before {
  content: "\f233";
}
.fa-user-plus:before {
  content: "\f234";
}
.fa-user-times:before {
  content: "\f235";
}
.fa-hotel:before,
.fa-bed:before {
  content: "\f236";
}
.fa-viacoin:before {
  content: "\f237";
}
.fa-train:before {
  content: "\f238";
}
.fa-subway:before {
  content: "\f239";
}
.fa-medium:before {
  content: "\f23a";
}
.fa-yc:before,
.fa-y-combinator:before {
  content: "\f23b";
}
.fa-optin-monster:before {
  content: "\f23c";
}
.fa-opencart:before {
  content: "\f23d";
}
.fa-expeditedssl:before {
  content: "\f23e";
}
.fa-battery-4:before,
.fa-battery:before,
.fa-battery-full:before {
  content: "\f240";
}
.fa-battery-3:before,
.fa-battery-three-quarters:before {
  content: "\f241";
}
.fa-battery-2:before,
.fa-battery-half:before {
  content: "\f242";
}
.fa-battery-1:before,
.fa-battery-quarter:before {
  content: "\f243";
}
.fa-battery-0:before,
.fa-battery-empty:before {
  content: "\f244";
}
.fa-mouse-pointer:before {
  content: "\f245";
}
.fa-i-cursor:before {
  content: "\f246";
}
.fa-object-group:before {
  content: "\f247";
}
.fa-object-ungroup:before {
  content: "\f248";
}
.fa-sticky-note:before {
  content: "\f249";
}
.fa-sticky-note-o:before {
  content: "\f24a";
}
.fa-cc-jcb:before {
  content: "\f24b";
}
.fa-cc-diners-club:before {
  content: "\f24c";
}
.fa-clone:before {
  content: "\f24d";
}
.fa-balance-scale:before {
  content: "\f24e";
}
.fa-hourglass-o:before {
  content: "\f250";
}
.fa-hourglass-1:before,
.fa-hourglass-start:before {
  content: "\f251";
}
.fa-hourglass-2:before,
.fa-hourglass-half:before {
  content: "\f252";
}
.fa-hourglass-3:before,
.fa-hourglass-end:before {
  content: "\f253";
}
.fa-hourglass:before {
  content: "\f254";
}
.fa-hand-grab-o:before,
.fa-hand-rock-o:before {
  content: "\f255";
}
.fa-hand-stop-o:before,
.fa-hand-paper-o:before {
  content: "\f256";
}
.fa-hand-scissors-o:before {
  content: "\f257";
}
.fa-hand-lizard-o:before {
  content: "\f258";
}
.fa-hand-spock-o:before {
  content: "\f259";
}
.fa-hand-pointer-o:before {
  content: "\f25a";
}
.fa-hand-peace-o:before {
  content: "\f25b";
}
.fa-trademark:before {
  content: "\f25c";
}
.fa-registered:before {
  content: "\f25d";
}
.fa-creative-commons:before {
  content: "\f25e";
}
.fa-gg:before {
  content: "\f260";
}
.fa-gg-circle:before {
  content: "\f261";
}
.fa-tripadvisor:before {
  content: "\f262";
}
.fa-odnoklassniki:before {
  content: "\f263";
}
.fa-odnoklassniki-square:before {
  content: "\f264";
}
.fa-get-pocket:before {
  content: "\f265";
}
.fa-wikipedia-w:before {
  content: "\f266";
}
.fa-safari:before {
  content: "\f267";
}
.fa-chrome:before {
  content: "\f268";
}
.fa-firefox:before {
  content: "\f269";
}
.fa-opera:before {
  content: "\f26a";
}
.fa-internet-explorer:before {
  content: "\f26b";
}
.fa-tv:before,
.fa-television:before {
  content: "\f26c";
}
.fa-contao:before {
  content: "\f26d";
}
.fa-500px:before {
  content: "\f26e";
}
.fa-amazon:before {
  content: "\f270";
}
.fa-calendar-plus-o:before {
  content: "\f271";
}
.fa-calendar-minus-o:before {
  content: "\f272";
}
.fa-calendar-times-o:before {
  content: "\f273";
}
.fa-calendar-check-o:before {
  content: "\f274";
}
.fa-industry:before {
  content: "\f275";
}
.fa-map-pin:before {
  content: "\f276";
}
.fa-map-signs:before {
  content: "\f277";
}
.fa-map-o:before {
  content: "\f278";
}
.fa-map:before {
  content: "\f279";
}
.fa-commenting:before {
  content: "\f27a";
}
.fa-commenting-o:before {
  content: "\f27b";
}
.fa-houzz:before {
  content: "\f27c";
}
.fa-vimeo:before {
  content: "\f27d";
}
.fa-black-tie:before {
  content: "\f27e";
}
.fa-fonticons:before {
  content: "\f280";
}
.fa-reddit-alien:before {
  content: "\f281";
}
.fa-edge:before {
  content: "\f282";
}
.fa-credit-card-alt:before {
  content: "\f283";
}
.fa-codiepie:before {
  content: "\f284";
}
.fa-modx:before {
  content: "\f285";
}
.fa-fort-awesome:before {
  content: "\f286";
}
.fa-usb:before {
  content: "\f287";
}
.fa-product-hunt:before {
  content: "\f288";
}
.fa-mixcloud:before {
  content: "\f289";
}
.fa-scribd:before {
  content: "\f28a";
}
.fa-pause-circle:before {
  content: "\f28b";
}
.fa-pause-circle-o:before {
  content: "\f28c";
}
.fa-stop-circle:before {
  content: "\f28d";
}
.fa-stop-circle-o:before {
  content: "\f28e";
}
.fa-shopping-bag:before {
  content: "\f290";
}
.fa-shopping-basket:before {
  content: "\f291";
}
.fa-hashtag:before {
  content: "\f292";
}
.fa-bluetooth:before {
  content: "\f293";
}
.fa-bluetooth-b:before {
  content: "\f294";
}
.fa-percent:before {
  content: "\f295";
}
.fa-gitlab:before {
  content: "\f296";
}
.fa-wpbeginner:before {
  content: "\f297";
}
.fa-wpforms:before {
  content: "\f298";
}
.fa-envira:before {
  content: "\f299";
}
.fa-universal-access:before {
  content: "\f29a";
}
.fa-wheelchair-alt:before {
  content: "\f29b";
}
.fa-question-circle-o:before {
  content: "\f29c";
}
.fa-blind:before {
  content: "\f29d";
}
.fa-audio-description:before {
  content: "\f29e";
}
.fa-volume-control-phone:before {
  content: "\f2a0";
}
.fa-braille:before {
  content: "\f2a1";
}
.fa-assistive-listening-systems:before {
  content: "\f2a2";
}
.fa-asl-interpreting:before,
.fa-american-sign-language-interpreting:before {
  content: "\f2a3";
}
.fa-deafness:before,
.fa-hard-of-hearing:before,
.fa-deaf:before {
  content: "\f2a4";
}
.fa-glide:before {
  content: "\f2a5";
}
.fa-glide-g:before {
  content: "\f2a6";
}
.fa-signing:before,
.fa-sign-language:before {
  content: "\f2a7";
}
.fa-low-vision:before {
  content: "\f2a8";
}
.fa-viadeo:before {
  content: "\f2a9";
}
.fa-viadeo-square:before {
  content: "\f2aa";
}
.fa-snapchat:before {
  content: "\f2ab";
}
.fa-snapchat-ghost:before {
  content: "\f2ac";
}
.fa-snapchat-square:before {
  content: "\f2ad";
}
.fa-pied-piper:before {
  content: "\f2ae";
}
.fa-first-order:before {
  content: "\f2b0";
}
.fa-yoast:before {
  content: "\f2b1";
}
.fa-themeisle:before {
  content: "\f2b2";
}
.fa-google-plus-circle:before,
.fa-google-plus-official:before {
  content: "\f2b3";
}
.fa-fa:before,
.fa-font-awesome:before {
  content: "\f2b4";
}
.fa-handshake-o:before {
  content: "\f2b5";
}
.fa-envelope-open:before {
  content: "\f2b6";
}
.fa-envelope-open-o:before {
  content: "\f2b7";
}
.fa-linode:before {
  content: "\f2b8";
}
.fa-address-book:before {
  content: "\f2b9";
}
.fa-address-book-o:before {
  content: "\f2ba";
}
.fa-vcard:before,
.fa-address-card:before {
  content: "\f2bb";
}
.fa-vcard-o:before,
.fa-address-card-o:before {
  content: "\f2bc";
}
.fa-user-circle:before {
  content: "\f2bd";
}
.fa-user-circle-o:before {
  content: "\f2be";
}
.fa-user-o:before {
  content: "\f2c0";
}
.fa-id-badge:before {
  content: "\f2c1";
}
.fa-drivers-license:before,
.fa-id-card:before {
  content: "\f2c2";
}
.fa-drivers-license-o:before,
.fa-id-card-o:before {
  content: "\f2c3";
}
.fa-quora:before {
  content: "\f2c4";
}
.fa-free-code-camp:before {
  content: "\f2c5";
}
.fa-telegram:before {
  content: "\f2c6";
}
.fa-thermometer-4:before,
.fa-thermometer:before,
.fa-thermometer-full:before {
  content: "\f2c7";
}
.fa-thermometer-3:before,
.fa-thermometer-three-quarters:before {
  content: "\f2c8";
}
.fa-thermometer-2:before,
.fa-thermometer-half:before {
  content: "\f2c9";
}
.fa-thermometer-1:before,
.fa-thermometer-quarter:before {
  content: "\f2ca";
}
.fa-thermometer-0:before,
.fa-thermometer-empty:before {
  content: "\f2cb";
}
.fa-shower:before {
  content: "\f2cc";
}
.fa-bathtub:before,
.fa-s15:before,
.fa-bath:before {
  content: "\f2cd";
}
.fa-podcast:before {
  content: "\f2ce";
}
.fa-window-maximize:before {
  content: "\f2d0";
}
.fa-window-minimize:before {
  content: "\f2d1";
}
.fa-window-restore:before {
  content: "\f2d2";
}
.fa-times-rectangle:before,
.fa-window-close:before {
  content: "\f2d3";
}
.fa-times-rectangle-o:before,
.fa-window-close-o:before {
  content: "\f2d4";
}
.fa-bandcamp:before {
  content: "\f2d5";
}
.fa-grav:before {
  content: "\f2d6";
}
.fa-etsy:before {
  content: "\f2d7";
}
.fa-imdb:before {
  content: "\f2d8";
}
.fa-ravelry:before {
  content: "\f2d9";
}
.fa-eercast:before {
  content: "\f2da";
}
.fa-microchip:before {
  content: "\f2db";
}
.fa-snowflake-o:before {
  content: "\f2dc";
}
.fa-superpowers:before {
  content: "\f2dd";
}
.fa-wpexplorer:before {
  content: "\f2de";
}
.fa-meetup:before {
  content: "\f2e0";
}
.sr-only {
  position: absolute;
  width: 1px;
  height: 1px;
  padding: 0;
  margin: -1px;
  overflow: hidden;
  clip: rect(0, 0, 0, 0);
  border: 0;
}
.sr-only-focusable:active,
.sr-only-focusable:focus {
  position: static;
  width: auto;
  height: auto;
  margin: 0;
  overflow: visible;
  clip: auto;
}
.sr-only-focusable:active,
.sr-only-focusable:focus {
  position: static;
  width: auto;
  height: auto;
  margin: 0;
  overflow: visible;
  clip: auto;
}
/*!
*
* IPython base
*
*/
.modal.fade .modal-dialog {
  -webkit-transform: translate(0, 0);
  -ms-transform: translate(0, 0);
  -o-transform: translate(0, 0);
  transform: translate(0, 0);
}
code {
  color: #000;
}
pre {
  font-size: inherit;
  line-height: inherit;
}
label {
  font-weight: normal;
}
/* Make the page background atleast 100% the height of the view port */
/* Make the page itself atleast 70% the height of the view port */
.border-box-sizing {
  box-sizing: border-box;
  -moz-box-sizing: border-box;
  -webkit-box-sizing: border-box;
}
.corner-all {
  border-radius: 2px;
}
.no-padding {
  padding: 0px;
}
/* Flexible box model classes */
/* Taken from Alex Russell http://infrequently.org/2009/08/css-3-progress/ */
/* This file is a compatability layer.  It allows the usage of flexible box 
model layouts accross multiple browsers, including older browsers.  The newest,
universal implementation of the flexible box model is used when available (see
`Modern browsers` comments below).  Browsers that are known to implement this 
new spec completely include:

    Firefox 28.0+
    Chrome 29.0+
    Internet Explorer 11+ 
    Opera 17.0+

Browsers not listed, including Safari, are supported via the styling under the
`Old browsers` comments below.
*/
.hbox {
  /* Old browsers */
  display: -webkit-box;
  -webkit-box-orient: horizontal;
  -webkit-box-align: stretch;
  display: -moz-box;
  -moz-box-orient: horizontal;
  -moz-box-align: stretch;
  display: box;
  box-orient: horizontal;
  box-align: stretch;
  /* Modern browsers */
  display: flex;
  flex-direction: row;
  align-items: stretch;
}
.hbox > * {
  /* Old browsers */
  -webkit-box-flex: 0;
  -moz-box-flex: 0;
  box-flex: 0;
  /* Modern browsers */
  flex: none;
}
.vbox {
  /* Old browsers */
  display: -webkit-box;
  -webkit-box-orient: vertical;
  -webkit-box-align: stretch;
  display: -moz-box;
  -moz-box-orient: vertical;
  -moz-box-align: stretch;
  display: box;
  box-orient: vertical;
  box-align: stretch;
  /* Modern browsers */
  display: flex;
  flex-direction: column;
  align-items: stretch;
}
.vbox > * {
  /* Old browsers */
  -webkit-box-flex: 0;
  -moz-box-flex: 0;
  box-flex: 0;
  /* Modern browsers */
  flex: none;
}
.hbox.reverse,
.vbox.reverse,
.reverse {
  /* Old browsers */
  -webkit-box-direction: reverse;
  -moz-box-direction: reverse;
  box-direction: reverse;
  /* Modern browsers */
  flex-direction: row-reverse;
}
.hbox.box-flex0,
.vbox.box-flex0,
.box-flex0 {
  /* Old browsers */
  -webkit-box-flex: 0;
  -moz-box-flex: 0;
  box-flex: 0;
  /* Modern browsers */
  flex: none;
  width: auto;
}
.hbox.box-flex1,
.vbox.box-flex1,
.box-flex1 {
  /* Old browsers */
  -webkit-box-flex: 1;
  -moz-box-flex: 1;
  box-flex: 1;
  /* Modern browsers */
  flex: 1;
}
.hbox.box-flex,
.vbox.box-flex,
.box-flex {
  /* Old browsers */
  /* Old browsers */
  -webkit-box-flex: 1;
  -moz-box-flex: 1;
  box-flex: 1;
  /* Modern browsers */
  flex: 1;
}
.hbox.box-flex2,
.vbox.box-flex2,
.box-flex2 {
  /* Old browsers */
  -webkit-box-flex: 2;
  -moz-box-flex: 2;
  box-flex: 2;
  /* Modern browsers */
  flex: 2;
}
.box-group1 {
  /*  Deprecated */
  -webkit-box-flex-group: 1;
  -moz-box-flex-group: 1;
  box-flex-group: 1;
}
.box-group2 {
  /* Deprecated */
  -webkit-box-flex-group: 2;
  -moz-box-flex-group: 2;
  box-flex-group: 2;
}
.hbox.start,
.vbox.start,
.start {
  /* Old browsers */
  -webkit-box-pack: start;
  -moz-box-pack: start;
  box-pack: start;
  /* Modern browsers */
  justify-content: flex-start;
}
.hbox.end,
.vbox.end,
.end {
  /* Old browsers */
  -webkit-box-pack: end;
  -moz-box-pack: end;
  box-pack: end;
  /* Modern browsers */
  justify-content: flex-end;
}
.hbox.center,
.vbox.center,
.center {
  /* Old browsers */
  -webkit-box-pack: center;
  -moz-box-pack: center;
  box-pack: center;
  /* Modern browsers */
  justify-content: center;
}
.hbox.baseline,
.vbox.baseline,
.baseline {
  /* Old browsers */
  -webkit-box-pack: baseline;
  -moz-box-pack: baseline;
  box-pack: baseline;
  /* Modern browsers */
  justify-content: baseline;
}
.hbox.stretch,
.vbox.stretch,
.stretch {
  /* Old browsers */
  -webkit-box-pack: stretch;
  -moz-box-pack: stretch;
  box-pack: stretch;
  /* Modern browsers */
  justify-content: stretch;
}
.hbox.align-start,
.vbox.align-start,
.align-start {
  /* Old browsers */
  -webkit-box-align: start;
  -moz-box-align: start;
  box-align: start;
  /* Modern browsers */
  align-items: flex-start;
}
.hbox.align-end,
.vbox.align-end,
.align-end {
  /* Old browsers */
  -webkit-box-align: end;
  -moz-box-align: end;
  box-align: end;
  /* Modern browsers */
  align-items: flex-end;
}
.hbox.align-center,
.vbox.align-center,
.align-center {
  /* Old browsers */
  -webkit-box-align: center;
  -moz-box-align: center;
  box-align: center;
  /* Modern browsers */
  align-items: center;
}
.hbox.align-baseline,
.vbox.align-baseline,
.align-baseline {
  /* Old browsers */
  -webkit-box-align: baseline;
  -moz-box-align: baseline;
  box-align: baseline;
  /* Modern browsers */
  align-items: baseline;
}
.hbox.align-stretch,
.vbox.align-stretch,
.align-stretch {
  /* Old browsers */
  -webkit-box-align: stretch;
  -moz-box-align: stretch;
  box-align: stretch;
  /* Modern browsers */
  align-items: stretch;
}
div.error {
  margin: 2em;
  text-align: center;
}
div.error > h1 {
  font-size: 500%;
  line-height: normal;
}
div.error > p {
  font-size: 200%;
  line-height: normal;
}
div.traceback-wrapper {
  text-align: left;
  max-width: 800px;
  margin: auto;
}
div.traceback-wrapper pre.traceback {
  max-height: 600px;
  overflow: auto;
}
/**
 * Primary styles
 *
 * Author: Jupyter Development Team
 */
body {
  background-color: #fff;
  /* This makes sure that the body covers the entire window and needs to
       be in a different element than the display: box in wrapper below */
  position: absolute;
  left: 0px;
  right: 0px;
  top: 0px;
  bottom: 0px;
  overflow: visible;
}
body > #header {
  /* Initially hidden to prevent FLOUC */
  display: none;
  background-color: #fff;
  /* Display over codemirror */
  position: relative;
  z-index: 100;
}
body > #header #header-container {
  display: flex;
  flex-direction: row;
  justify-content: space-between;
  padding: 5px;
  padding-bottom: 5px;
  padding-top: 5px;
  box-sizing: border-box;
  -moz-box-sizing: border-box;
  -webkit-box-sizing: border-box;
}
body > #header .header-bar {
  width: 100%;
  height: 1px;
  background: #e7e7e7;
  margin-bottom: -1px;
}
@media print {
  body > #header {
    display: none !important;
  }
}
#header-spacer {
  width: 100%;
  visibility: hidden;
}
@media print {
  #header-spacer {
    display: none;
  }
}
#ipython_notebook {
  padding-left: 0px;
  padding-top: 1px;
  padding-bottom: 1px;
}
[dir="rtl"] #ipython_notebook {
  margin-right: 10px;
  margin-left: 0;
}
[dir="rtl"] #ipython_notebook.pull-left {
  float: right !important;
  float: right;
}
.flex-spacer {
  flex: 1;
}
#noscript {
  width: auto;
  padding-top: 16px;
  padding-bottom: 16px;
  text-align: center;
  font-size: 22px;
  color: red;
  font-weight: bold;
}
#ipython_notebook img {
  height: 28px;
}
#site {
  width: 100%;
  display: none;
  box-sizing: border-box;
  -moz-box-sizing: border-box;
  -webkit-box-sizing: border-box;
  overflow: auto;
}
@media print {
  #site {
    height: auto !important;
  }
}
/* Smaller buttons */
.ui-button .ui-button-text {
  padding: 0.2em 0.8em;
  font-size: 77%;
}
input.ui-button {
  padding: 0.3em 0.9em;
}
span#kernel_logo_widget {
  margin: 0 10px;
}
span#login_widget {
  float: right;
}
[dir="rtl"] span#login_widget {
  float: left;
}
span#login_widget > .button,
#logout {
  color: #333;
  background-color: #fff;
  border-color: #ccc;
}
span#login_widget > .button:focus,
#logout:focus,
span#login_widget > .button.focus,
#logout.focus {
  color: #333;
  background-color: #e6e6e6;
  border-color: #8c8c8c;
}
span#login_widget > .button:hover,
#logout:hover {
  color: #333;
  background-color: #e6e6e6;
  border-color: #adadad;
}
span#login_widget > .button:active,
#logout:active,
span#login_widget > .button.active,
#logout.active,
.open > .dropdown-togglespan#login_widget > .button,
.open > .dropdown-toggle#logout {
  color: #333;
  background-color: #e6e6e6;
  border-color: #adadad;
}
span#login_widget > .button:active:hover,
#logout:active:hover,
span#login_widget > .button.active:hover,
#logout.active:hover,
.open > .dropdown-togglespan#login_widget > .button:hover,
.open > .dropdown-toggle#logout:hover,
span#login_widget > .button:active:focus,
#logout:active:focus,
span#login_widget > .button.active:focus,
#logout.active:focus,
.open > .dropdown-togglespan#login_widget > .button:focus,
.open > .dropdown-toggle#logout:focus,
span#login_widget > .button:active.focus,
#logout:active.focus,
span#login_widget > .button.active.focus,
#logout.active.focus,
.open > .dropdown-togglespan#login_widget > .button.focus,
.open > .dropdown-toggle#logout.focus {
  color: #333;
  background-color: #d4d4d4;
  border-color: #8c8c8c;
}
span#login_widget > .button:active,
#logout:active,
span#login_widget > .button.active,
#logout.active,
.open > .dropdown-togglespan#login_widget > .button,
.open > .dropdown-toggle#logout {
  background-image: none;
}
span#login_widget > .button.disabled:hover,
#logout.disabled:hover,
span#login_widget > .button[disabled]:hover,
#logout[disabled]:hover,
fieldset[disabled] span#login_widget > .button:hover,
fieldset[disabled] #logout:hover,
span#login_widget > .button.disabled:focus,
#logout.disabled:focus,
span#login_widget > .button[disabled]:focus,
#logout[disabled]:focus,
fieldset[disabled] span#login_widget > .button:focus,
fieldset[disabled] #logout:focus,
span#login_widget > .button.disabled.focus,
#logout.disabled.focus,
span#login_widget > .button[disabled].focus,
#logout[disabled].focus,
fieldset[disabled] span#login_widget > .button.focus,
fieldset[disabled] #logout.focus {
  background-color: #fff;
  border-color: #ccc;
}
span#login_widget > .button .badge,
#logout .badge {
  color: #fff;
  background-color: #333;
}
.nav-header {
  text-transform: none;
}
#header > span {
  margin-top: 10px;
}
.modal_stretch .modal-dialog {
  /* Old browsers */
  display: -webkit-box;
  -webkit-box-orient: vertical;
  -webkit-box-align: stretch;
  display: -moz-box;
  -moz-box-orient: vertical;
  -moz-box-align: stretch;
  display: box;
  box-orient: vertical;
  box-align: stretch;
  /* Modern browsers */
  display: flex;
  flex-direction: column;
  align-items: stretch;
  min-height: 80vh;
}
.modal_stretch .modal-dialog .modal-body {
  max-height: calc(100vh - 200px);
  overflow: auto;
  flex: 1;
}
.modal-header {
  cursor: move;
}
@media (min-width: 768px) {
  .modal .modal-dialog {
    width: 700px;
  }
}
@media (min-width: 768px) {
  select.form-control {
    margin-left: 12px;
    margin-right: 12px;
  }
}
/*!
*
* IPython auth
*
*/
.center-nav {
  display: inline-block;
  margin-bottom: -4px;
}
[dir="rtl"] .center-nav form.pull-left {
  float: right !important;
  float: right;
}
[dir="rtl"] .center-nav .navbar-text {
  float: right;
}
[dir="rtl"] .navbar-inner {
  text-align: right;
}
[dir="rtl"] div.text-left {
  text-align: right;
}
/*!
*
* IPython tree view
*
*/
/* We need an invisible input field on top of the sentense*/
/* "Drag file onto the list ..." */
.alternate_upload {
  background-color: none;
  display: inline;
}
.alternate_upload.form {
  padding: 0;
  margin: 0;
}
.alternate_upload input.fileinput {
  position: absolute;
  display: block;
  width: 100%;
  height: 100%;
  overflow: hidden;
  cursor: pointer;
  opacity: 0;
  z-index: 2;
}
.alternate_upload .btn-xs > input.fileinput {
  margin: -1px -5px;
}
.alternate_upload .btn-upload {
  position: relative;
  height: 22px;
}
::-webkit-file-upload-button {
  cursor: pointer;
}
/**
 * Primary styles
 *
 * Author: Jupyter Development Team
 */
ul#tabs {
  margin-bottom: 4px;
}
ul#tabs a {
  padding-top: 6px;
  padding-bottom: 4px;
}
[dir="rtl"] ul#tabs.nav-tabs > li {
  float: right;
}
[dir="rtl"] ul#tabs.nav.nav-tabs {
  padding-right: 0;
}
ul.breadcrumb a:focus,
ul.breadcrumb a:hover {
  text-decoration: none;
}
ul.breadcrumb i.icon-home {
  font-size: 16px;
  margin-right: 4px;
}
ul.breadcrumb span {
  color: #5e5e5e;
}
.list_toolbar {
  padding: 4px 0 4px 0;
  vertical-align: middle;
}
.list_toolbar .tree-buttons {
  padding-top: 1px;
}
[dir="rtl"] .list_toolbar .tree-buttons .pull-right {
  float: left !important;
  float: left;
}
[dir="rtl"] .list_toolbar .col-sm-4,
[dir="rtl"] .list_toolbar .col-sm-8 {
  float: right;
}
.dynamic-buttons {
  padding-top: 3px;
  display: inline-block;
}
.list_toolbar [class*="span"] {
  min-height: 24px;
}
.list_header {
  font-weight: bold;
  background-color: #EEE;
}
.list_placeholder {
  font-weight: bold;
  padding-top: 4px;
  padding-bottom: 4px;
  padding-left: 7px;
  padding-right: 7px;
}
.list_container {
  margin-top: 4px;
  margin-bottom: 20px;
  border: 1px solid #ddd;
  border-radius: 2px;
}
.list_container > div {
  border-bottom: 1px solid #ddd;
}
.list_container > div:hover .list-item {
  background-color: red;
}
.list_container > div:last-child {
  border: none;
}
.list_item:hover .list_item {
  background-color: #ddd;
}
.list_item a {
  text-decoration: none;
}
.list_item:hover {
  background-color: #fafafa;
}
.list_header > div,
.list_item > div {
  padding-top: 4px;
  padding-bottom: 4px;
  padding-left: 7px;
  padding-right: 7px;
  line-height: 22px;
}
.list_header > div input,
.list_item > div input {
  margin-right: 7px;
  margin-left: 14px;
  vertical-align: text-bottom;
  line-height: 22px;
  position: relative;
  top: -1px;
}
.list_header > div .item_link,
.list_item > div .item_link {
  margin-left: -1px;
  vertical-align: baseline;
  line-height: 22px;
}
[dir="rtl"] .list_item > div input {
  margin-right: 0;
}
.new-file input[type=checkbox] {
  visibility: hidden;
}
.item_name {
  line-height: 22px;
  height: 24px;
}
.item_icon {
  font-size: 14px;
  color: #5e5e5e;
  margin-right: 7px;
  margin-left: 7px;
  line-height: 22px;
  vertical-align: baseline;
}
.item_modified {
  margin-right: 7px;
  margin-left: 7px;
}
[dir="rtl"] .item_modified.pull-right {
  float: left !important;
  float: left;
}
.item_buttons {
  line-height: 1em;
  margin-left: -5px;
}
.item_buttons .btn,
.item_buttons .btn-group,
.item_buttons .input-group {
  float: left;
}
.item_buttons > .btn,
.item_buttons > .btn-group,
.item_buttons > .input-group {
  margin-left: 5px;
}
.item_buttons .btn {
  min-width: 13ex;
}
.item_buttons .running-indicator {
  padding-top: 4px;
  color: #5cb85c;
}
.item_buttons .kernel-name {
  padding-top: 4px;
  color: #5bc0de;
  margin-right: 7px;
  float: left;
}
[dir="rtl"] .item_buttons.pull-right {
  float: left !important;
  float: left;
}
[dir="rtl"] .item_buttons .kernel-name {
  margin-left: 7px;
  float: right;
}
.toolbar_info {
  height: 24px;
  line-height: 24px;
}
.list_item input:not([type=checkbox]) {
  padding-top: 3px;
  padding-bottom: 3px;
  height: 22px;
  line-height: 14px;
  margin: 0px;
}
.highlight_text {
  color: blue;
}
#project_name {
  display: inline-block;
  padding-left: 7px;
  margin-left: -2px;
}
#project_name > .breadcrumb {
  padding: 0px;
  margin-bottom: 0px;
  background-color: transparent;
  font-weight: bold;
}
.sort_button {
  display: inline-block;
  padding-left: 7px;
}
[dir="rtl"] .sort_button.pull-right {
  float: left !important;
  float: left;
}
#tree-selector {
  padding-right: 0px;
}
#button-select-all {
  min-width: 50px;
}
[dir="rtl"] #button-select-all.btn {
  float: right ;
}
#select-all {
  margin-left: 7px;
  margin-right: 2px;
  margin-top: 2px;
  height: 16px;
}
[dir="rtl"] #select-all.pull-left {
  float: right !important;
  float: right;
}
.menu_icon {
  margin-right: 2px;
}
.tab-content .row {
  margin-left: 0px;
  margin-right: 0px;
}
.folder_icon:before {
  display: inline-block;
  font: normal normal normal 14px/1 FontAwesome;
  font-size: inherit;
  text-rendering: auto;
  -webkit-font-smoothing: antialiased;
  -moz-osx-font-smoothing: grayscale;
  content: "\f114";
}
.folder_icon:before.fa-pull-left {
  margin-right: .3em;
}
.folder_icon:before.fa-pull-right {
  margin-left: .3em;
}
.folder_icon:before.pull-left {
  margin-right: .3em;
}
.folder_icon:before.pull-right {
  margin-left: .3em;
}
.notebook_icon:before {
  display: inline-block;
  font: normal normal normal 14px/1 FontAwesome;
  font-size: inherit;
  text-rendering: auto;
  -webkit-font-smoothing: antialiased;
  -moz-osx-font-smoothing: grayscale;
  content: "\f02d";
  position: relative;
  top: -1px;
}
.notebook_icon:before.fa-pull-left {
  margin-right: .3em;
}
.notebook_icon:before.fa-pull-right {
  margin-left: .3em;
}
.notebook_icon:before.pull-left {
  margin-right: .3em;
}
.notebook_icon:before.pull-right {
  margin-left: .3em;
}
.running_notebook_icon:before {
  display: inline-block;
  font: normal normal normal 14px/1 FontAwesome;
  font-size: inherit;
  text-rendering: auto;
  -webkit-font-smoothing: antialiased;
  -moz-osx-font-smoothing: grayscale;
  content: "\f02d";
  position: relative;
  top: -1px;
  color: #5cb85c;
}
.running_notebook_icon:before.fa-pull-left {
  margin-right: .3em;
}
.running_notebook_icon:before.fa-pull-right {
  margin-left: .3em;
}
.running_notebook_icon:before.pull-left {
  margin-right: .3em;
}
.running_notebook_icon:before.pull-right {
  margin-left: .3em;
}
.file_icon:before {
  display: inline-block;
  font: normal normal normal 14px/1 FontAwesome;
  font-size: inherit;
  text-rendering: auto;
  -webkit-font-smoothing: antialiased;
  -moz-osx-font-smoothing: grayscale;
  content: "\f016";
  position: relative;
  top: -2px;
}
.file_icon:before.fa-pull-left {
  margin-right: .3em;
}
.file_icon:before.fa-pull-right {
  margin-left: .3em;
}
.file_icon:before.pull-left {
  margin-right: .3em;
}
.file_icon:before.pull-right {
  margin-left: .3em;
}
#notebook_toolbar .pull-right {
  padding-top: 0px;
  margin-right: -1px;
}
ul#new-menu {
  left: auto;
  right: 0;
}
#new-menu .dropdown-header {
  font-size: 10px;
  border-bottom: 1px solid #e5e5e5;
  padding: 0 0 3px;
  margin: -3px 20px 0;
}
.kernel-menu-icon {
  padding-right: 12px;
  width: 24px;
  content: "\f096";
}
.kernel-menu-icon:before {
  content: "\f096";
}
.kernel-menu-icon-current:before {
  content: "\f00c";
}
#tab_content {
  padding-top: 20px;
}
#running .panel-group .panel {
  margin-top: 3px;
  margin-bottom: 1em;
}
#running .panel-group .panel .panel-heading {
  background-color: #EEE;
  padding-top: 4px;
  padding-bottom: 4px;
  padding-left: 7px;
  padding-right: 7px;
  line-height: 22px;
}
#running .panel-group .panel .panel-heading a:focus,
#running .panel-group .panel .panel-heading a:hover {
  text-decoration: none;
}
#running .panel-group .panel .panel-body {
  padding: 0px;
}
#running .panel-group .panel .panel-body .list_container {
  margin-top: 0px;
  margin-bottom: 0px;
  border: 0px;
  border-radius: 0px;
}
#running .panel-group .panel .panel-body .list_container .list_item {
  border-bottom: 1px solid #ddd;
}
#running .panel-group .panel .panel-body .list_container .list_item:last-child {
  border-bottom: 0px;
}
.delete-button {
  display: none;
}
.duplicate-button {
  display: none;
}
.rename-button {
  display: none;
}
.move-button {
  display: none;
}
.download-button {
  display: none;
}
.shutdown-button {
  display: none;
}
.dynamic-instructions {
  display: inline-block;
  padding-top: 4px;
}
/*!
*
* IPython text editor webapp
*
*/
.selected-keymap i.fa {
  padding: 0px 5px;
}
.selected-keymap i.fa:before {
  content: "\f00c";
}
#mode-menu {
  overflow: auto;
  max-height: 20em;
}
.edit_app #header {
  -webkit-box-shadow: 0px 0px 12px 1px rgba(87, 87, 87, 0.2);
  box-shadow: 0px 0px 12px 1px rgba(87, 87, 87, 0.2);
}
.edit_app #menubar .navbar {
  /* Use a negative 1 bottom margin, so the border overlaps the border of the
    header */
  margin-bottom: -1px;
}
.dirty-indicator {
  display: inline-block;
  font: normal normal normal 14px/1 FontAwesome;
  font-size: inherit;
  text-rendering: auto;
  -webkit-font-smoothing: antialiased;
  -moz-osx-font-smoothing: grayscale;
  width: 20px;
}
.dirty-indicator.fa-pull-left {
  margin-right: .3em;
}
.dirty-indicator.fa-pull-right {
  margin-left: .3em;
}
.dirty-indicator.pull-left {
  margin-right: .3em;
}
.dirty-indicator.pull-right {
  margin-left: .3em;
}
.dirty-indicator-dirty {
  display: inline-block;
  font: normal normal normal 14px/1 FontAwesome;
  font-size: inherit;
  text-rendering: auto;
  -webkit-font-smoothing: antialiased;
  -moz-osx-font-smoothing: grayscale;
  width: 20px;
}
.dirty-indicator-dirty.fa-pull-left {
  margin-right: .3em;
}
.dirty-indicator-dirty.fa-pull-right {
  margin-left: .3em;
}
.dirty-indicator-dirty.pull-left {
  margin-right: .3em;
}
.dirty-indicator-dirty.pull-right {
  margin-left: .3em;
}
.dirty-indicator-clean {
  display: inline-block;
  font: normal normal normal 14px/1 FontAwesome;
  font-size: inherit;
  text-rendering: auto;
  -webkit-font-smoothing: antialiased;
  -moz-osx-font-smoothing: grayscale;
  width: 20px;
}
.dirty-indicator-clean.fa-pull-left {
  margin-right: .3em;
}
.dirty-indicator-clean.fa-pull-right {
  margin-left: .3em;
}
.dirty-indicator-clean.pull-left {
  margin-right: .3em;
}
.dirty-indicator-clean.pull-right {
  margin-left: .3em;
}
.dirty-indicator-clean:before {
  display: inline-block;
  font: normal normal normal 14px/1 FontAwesome;
  font-size: inherit;
  text-rendering: auto;
  -webkit-font-smoothing: antialiased;
  -moz-osx-font-smoothing: grayscale;
  content: "\f00c";
}
.dirty-indicator-clean:before.fa-pull-left {
  margin-right: .3em;
}
.dirty-indicator-clean:before.fa-pull-right {
  margin-left: .3em;
}
.dirty-indicator-clean:before.pull-left {
  margin-right: .3em;
}
.dirty-indicator-clean:before.pull-right {
  margin-left: .3em;
}
#filename {
  font-size: 16pt;
  display: table;
  padding: 0px 5px;
}
#current-mode {
  padding-left: 5px;
  padding-right: 5px;
}
#texteditor-backdrop {
  padding-top: 20px;
  padding-bottom: 20px;
}
@media not print {
  #texteditor-backdrop {
    background-color: #EEE;
  }
}
@media print {
  #texteditor-backdrop #texteditor-container .CodeMirror-gutter,
  #texteditor-backdrop #texteditor-container .CodeMirror-gutters {
    background-color: #fff;
  }
}
@media not print {
  #texteditor-backdrop #texteditor-container .CodeMirror-gutter,
  #texteditor-backdrop #texteditor-container .CodeMirror-gutters {
    background-color: #fff;
  }
}
@media not print {
  #texteditor-backdrop #texteditor-container {
    padding: 0px;
    background-color: #fff;
    -webkit-box-shadow: 0px 0px 12px 1px rgba(87, 87, 87, 0.2);
    box-shadow: 0px 0px 12px 1px rgba(87, 87, 87, 0.2);
  }
}
.CodeMirror-dialog {
  background-color: #fff;
}
/*!
*
* IPython notebook
*
*/
/* CSS font colors for translated ANSI escape sequences */
/* The color values are a mix of
   http://www.xcolors.net/dl/baskerville-ivorylight and
   http://www.xcolors.net/dl/euphrasia */
.ansi-black-fg {
  color: #3E424D;
}
.ansi-black-bg {
  background-color: #3E424D;
}
.ansi-black-intense-fg {
  color: #282C36;
}
.ansi-black-intense-bg {
  background-color: #282C36;
}
.ansi-red-fg {
  color: #E75C58;
}
.ansi-red-bg {
  background-color: #E75C58;
}
.ansi-red-intense-fg {
  color: #B22B31;
}
.ansi-red-intense-bg {
  background-color: #B22B31;
}
.ansi-green-fg {
  color: #00A250;
}
.ansi-green-bg {
  background-color: #00A250;
}
.ansi-green-intense-fg {
  color: #007427;
}
.ansi-green-intense-bg {
  background-color: #007427;
}
.ansi-yellow-fg {
  color: #DDB62B;
}
.ansi-yellow-bg {
  background-color: #DDB62B;
}
.ansi-yellow-intense-fg {
  color: #B27D12;
}
.ansi-yellow-intense-bg {
  background-color: #B27D12;
}
.ansi-blue-fg {
  color: #208FFB;
}
.ansi-blue-bg {
  background-color: #208FFB;
}
.ansi-blue-intense-fg {
  color: #0065CA;
}
.ansi-blue-intense-bg {
  background-color: #0065CA;
}
.ansi-magenta-fg {
  color: #D160C4;
}
.ansi-magenta-bg {
  background-color: #D160C4;
}
.ansi-magenta-intense-fg {
  color: #A03196;
}
.ansi-magenta-intense-bg {
  background-color: #A03196;
}
.ansi-cyan-fg {
  color: #60C6C8;
}
.ansi-cyan-bg {
  background-color: #60C6C8;
}
.ansi-cyan-intense-fg {
  color: #258F8F;
}
.ansi-cyan-intense-bg {
  background-color: #258F8F;
}
.ansi-white-fg {
  color: #C5C1B4;
}
.ansi-white-bg {
  background-color: #C5C1B4;
}
.ansi-white-intense-fg {
  color: #A1A6B2;
}
.ansi-white-intense-bg {
  background-color: #A1A6B2;
}
.ansi-default-inverse-fg {
  color: #FFFFFF;
}
.ansi-default-inverse-bg {
  background-color: #000000;
}
.ansi-bold {
  font-weight: bold;
}
.ansi-underline {
  text-decoration: underline;
}
/* The following styles are deprecated an will be removed in a future version */
.ansibold {
  font-weight: bold;
}
.ansi-inverse {
  outline: 0.5px dotted;
}
/* use dark versions for foreground, to improve visibility */
.ansiblack {
  color: black;
}
.ansired {
  color: darkred;
}
.ansigreen {
  color: darkgreen;
}
.ansiyellow {
  color: #c4a000;
}
.ansiblue {
  color: darkblue;
}
.ansipurple {
  color: darkviolet;
}
.ansicyan {
  color: steelblue;
}
.ansigray {
  color: gray;
}
/* and light for background, for the same reason */
.ansibgblack {
  background-color: black;
}
.ansibgred {
  background-color: red;
}
.ansibggreen {
  background-color: green;
}
.ansibgyellow {
  background-color: yellow;
}
.ansibgblue {
  background-color: blue;
}
.ansibgpurple {
  background-color: magenta;
}
.ansibgcyan {
  background-color: cyan;
}
.ansibggray {
  background-color: gray;
}
div.cell {
  /* Old browsers */
  display: -webkit-box;
  -webkit-box-orient: vertical;
  -webkit-box-align: stretch;
  display: -moz-box;
  -moz-box-orient: vertical;
  -moz-box-align: stretch;
  display: box;
  box-orient: vertical;
  box-align: stretch;
  /* Modern browsers */
  display: flex;
  flex-direction: column;
  align-items: stretch;
  border-radius: 2px;
  box-sizing: border-box;
  -moz-box-sizing: border-box;
  -webkit-box-sizing: border-box;
  border-width: 1px;
  border-style: solid;
  border-color: transparent;
  width: 100%;
  padding: 5px;
  /* This acts as a spacer between cells, that is outside the border */
  margin: 0px;
  outline: none;
  position: relative;
  overflow: visible;
}
div.cell:before {
  position: absolute;
  display: block;
  top: -1px;
  left: -1px;
  width: 5px;
  height: calc(100% +  2px);
  content: '';
  background: transparent;
}
div.cell.jupyter-soft-selected {
  border-left-color: #E3F2FD;
  border-left-width: 1px;
  padding-left: 5px;
  border-right-color: #E3F2FD;
  border-right-width: 1px;
  background: #E3F2FD;
}
@media print {
  div.cell.jupyter-soft-selected {
    border-color: transparent;
  }
}
div.cell.selected,
div.cell.selected.jupyter-soft-selected {
  border-color: #ababab;
}
div.cell.selected:before,
div.cell.selected.jupyter-soft-selected:before {
  position: absolute;
  display: block;
  top: -1px;
  left: -1px;
  width: 5px;
  height: calc(100% +  2px);
  content: '';
  background: #42A5F5;
}
@media print {
  div.cell.selected,
  div.cell.selected.jupyter-soft-selected {
    border-color: transparent;
  }
}
.edit_mode div.cell.selected {
  border-color: #66BB6A;
}
.edit_mode div.cell.selected:before {
  position: absolute;
  display: block;
  top: -1px;
  left: -1px;
  width: 5px;
  height: calc(100% +  2px);
  content: '';
  background: #66BB6A;
}
@media print {
  .edit_mode div.cell.selected {
    border-color: transparent;
  }
}
.prompt {
  /* This needs to be wide enough for 3 digit prompt numbers: In[100]: */
  min-width: 14ex;
  /* This padding is tuned to match the padding on the CodeMirror editor. */
  padding: 0.4em;
  margin: 0px;
  font-family: monospace;
  text-align: right;
  /* This has to match that of the the CodeMirror class line-height below */
  line-height: 1.21429em;
  /* Don't highlight prompt number selection */
  -webkit-touch-callout: none;
  -webkit-user-select: none;
  -khtml-user-select: none;
  -moz-user-select: none;
  -ms-user-select: none;
  user-select: none;
  /* Use default cursor */
  cursor: default;
}
@media (max-width: 540px) {
  .prompt {
    text-align: left;
  }
}
div.inner_cell {
  min-width: 0;
  /* Old browsers */
  display: -webkit-box;
  -webkit-box-orient: vertical;
  -webkit-box-align: stretch;
  display: -moz-box;
  -moz-box-orient: vertical;
  -moz-box-align: stretch;
  display: box;
  box-orient: vertical;
  box-align: stretch;
  /* Modern browsers */
  display: flex;
  flex-direction: column;
  align-items: stretch;
  /* Old browsers */
  -webkit-box-flex: 1;
  -moz-box-flex: 1;
  box-flex: 1;
  /* Modern browsers */
  flex: 1;
}
/* input_area and input_prompt must match in top border and margin for alignment */
div.input_area {
  border: 1px solid #cfcfcf;
  border-radius: 2px;
  background: #f7f7f7;
  line-height: 1.21429em;
}
/* This is needed so that empty prompt areas can collapse to zero height when there
   is no content in the output_subarea and the prompt. The main purpose of this is
   to make sure that empty JavaScript output_subareas have no height. */
div.prompt:empty {
  padding-top: 0;
  padding-bottom: 0;
}
div.unrecognized_cell {
  padding: 5px 5px 5px 0px;
  /* Old browsers */
  display: -webkit-box;
  -webkit-box-orient: horizontal;
  -webkit-box-align: stretch;
  display: -moz-box;
  -moz-box-orient: horizontal;
  -moz-box-align: stretch;
  display: box;
  box-orient: horizontal;
  box-align: stretch;
  /* Modern browsers */
  display: flex;
  flex-direction: row;
  align-items: stretch;
}
div.unrecognized_cell .inner_cell {
  border-radius: 2px;
  padding: 5px;
  font-weight: bold;
  color: red;
  border: 1px solid #cfcfcf;
  background: #eaeaea;
}
div.unrecognized_cell .inner_cell a {
  color: inherit;
  text-decoration: none;
}
div.unrecognized_cell .inner_cell a:hover {
  color: inherit;
  text-decoration: none;
}
@media (max-width: 540px) {
  div.unrecognized_cell > div.prompt {
    display: none;
  }
}
div.code_cell {
  /* avoid page breaking on code cells when printing */
}
@media print {
  div.code_cell {
    page-break-inside: avoid;
  }
}
/* any special styling for code cells that are currently running goes here */
div.input {
  page-break-inside: avoid;
  /* Old browsers */
  display: -webkit-box;
  -webkit-box-orient: horizontal;
  -webkit-box-align: stretch;
  display: -moz-box;
  -moz-box-orient: horizontal;
  -moz-box-align: stretch;
  display: box;
  box-orient: horizontal;
  box-align: stretch;
  /* Modern browsers */
  display: flex;
  flex-direction: row;
  align-items: stretch;
}
@media (max-width: 540px) {
  div.input {
    /* Old browsers */
    display: -webkit-box;
    -webkit-box-orient: vertical;
    -webkit-box-align: stretch;
    display: -moz-box;
    -moz-box-orient: vertical;
    -moz-box-align: stretch;
    display: box;
    box-orient: vertical;
    box-align: stretch;
    /* Modern browsers */
    display: flex;
    flex-direction: column;
    align-items: stretch;
  }
}
/* input_area and input_prompt must match in top border and margin for alignment */
div.input_prompt {
  color: #303F9F;
  border-top: 1px solid transparent;
}
div.input_area > div.highlight {
  margin: 0.4em;
  border: none;
  padding: 0px;
  background-color: transparent;
}
div.input_area > div.highlight > pre {
  margin: 0px;
  border: none;
  padding: 0px;
  background-color: transparent;
}
/* The following gets added to the <head> if it is detected that the user has a
 * monospace font with inconsistent normal/bold/italic height.  See
 * notebookmain.js.  Such fonts will have keywords vertically offset with
 * respect to the rest of the text.  The user should select a better font.
 * See: https://github.com/ipython/ipython/issues/1503
 *
 * .CodeMirror span {
 *      vertical-align: bottom;
 * }
 */
.CodeMirror {
  line-height: 1.21429em;
  /* Changed from 1em to our global default */
  font-size: 14px;
  height: auto;
  /* Changed to auto to autogrow */
  background: none;
  /* Changed from white to allow our bg to show through */
}
.CodeMirror-scroll {
  /*  The CodeMirror docs are a bit fuzzy on if overflow-y should be hidden or visible.*/
  /*  We have found that if it is visible, vertical scrollbars appear with font size changes.*/
  overflow-y: hidden;
  overflow-x: auto;
}
.CodeMirror-lines {
  /* In CM2, this used to be 0.4em, but in CM3 it went to 4px. We need the em value because */
  /* we have set a different line-height and want this to scale with that. */
  /* Note that this should set vertical padding only, since CodeMirror assumes
       that horizontal padding will be set on CodeMirror pre */
  padding: 0.4em 0;
}
.CodeMirror-linenumber {
  padding: 0 8px 0 4px;
}
.CodeMirror-gutters {
  border-bottom-left-radius: 2px;
  border-top-left-radius: 2px;
}
.CodeMirror pre {
  /* In CM3 this went to 4px from 0 in CM2. This sets horizontal padding only,
    use .CodeMirror-lines for vertical */
  padding: 0 0.4em;
  border: 0;
  border-radius: 0;
}
.CodeMirror-cursor {
  border-left: 1.4px solid black;
}
@media screen and (min-width: 2138px) and (max-width: 4319px) {
  .CodeMirror-cursor {
    border-left: 2px solid black;
  }
}
@media screen and (min-width: 4320px) {
  .CodeMirror-cursor {
    border-left: 4px solid black;
  }
}
/*

Original style from softwaremaniacs.org (c) Ivan Sagalaev <Maniac@SoftwareManiacs.Org>
Adapted from GitHub theme

*/
.highlight-base {
  color: #000;
}
.highlight-variable {
  color: #000;
}
.highlight-variable-2 {
  color: #1a1a1a;
}
.highlight-variable-3 {
  color: #333333;
}
.highlight-string {
  color: #BA2121;
}
.highlight-comment {
  color: #408080;
  font-style: italic;
}
.highlight-number {
  color: #080;
}
.highlight-atom {
  color: #88F;
}
.highlight-keyword {
  color: #008000;
  font-weight: bold;
}
.highlight-builtin {
  color: #008000;
}
.highlight-error {
  color: #f00;
}
.highlight-operator {
  color: #AA22FF;
  font-weight: bold;
}
.highlight-meta {
  color: #AA22FF;
}
/* previously not defined, copying from default codemirror */
.highlight-def {
  color: #00f;
}
.highlight-string-2 {
  color: #f50;
}
.highlight-qualifier {
  color: #555;
}
.highlight-bracket {
  color: #997;
}
.highlight-tag {
  color: #170;
}
.highlight-attribute {
  color: #00c;
}
.highlight-header {
  color: blue;
}
.highlight-quote {
  color: #090;
}
.highlight-link {
  color: #00c;
}
/* apply the same style to codemirror */
.cm-s-ipython span.cm-keyword {
  color: #008000;
  font-weight: bold;
}
.cm-s-ipython span.cm-atom {
  color: #88F;
}
.cm-s-ipython span.cm-number {
  color: #080;
}
.cm-s-ipython span.cm-def {
  color: #00f;
}
.cm-s-ipython span.cm-variable {
  color: #000;
}
.cm-s-ipython span.cm-operator {
  color: #AA22FF;
  font-weight: bold;
}
.cm-s-ipython span.cm-variable-2 {
  color: #1a1a1a;
}
.cm-s-ipython span.cm-variable-3 {
  color: #333333;
}
.cm-s-ipython span.cm-comment {
  color: #408080;
  font-style: italic;
}
.cm-s-ipython span.cm-string {
  color: #BA2121;
}
.cm-s-ipython span.cm-string-2 {
  color: #f50;
}
.cm-s-ipython span.cm-meta {
  color: #AA22FF;
}
.cm-s-ipython span.cm-qualifier {
  color: #555;
}
.cm-s-ipython span.cm-builtin {
  color: #008000;
}
.cm-s-ipython span.cm-bracket {
  color: #997;
}
.cm-s-ipython span.cm-tag {
  color: #170;
}
.cm-s-ipython span.cm-attribute {
  color: #00c;
}
.cm-s-ipython span.cm-header {
  color: blue;
}
.cm-s-ipython span.cm-quote {
  color: #090;
}
.cm-s-ipython span.cm-link {
  color: #00c;
}
.cm-s-ipython span.cm-error {
  color: #f00;
}
.cm-s-ipython span.cm-tab {
  background: url();
  background-position: right;
  background-repeat: no-repeat;
}
div.output_wrapper {
  /* this position must be relative to enable descendents to be absolute within it */
  position: relative;
  /* Old browsers */
  display: -webkit-box;
  -webkit-box-orient: vertical;
  -webkit-box-align: stretch;
  display: -moz-box;
  -moz-box-orient: vertical;
  -moz-box-align: stretch;
  display: box;
  box-orient: vertical;
  box-align: stretch;
  /* Modern browsers */
  display: flex;
  flex-direction: column;
  align-items: stretch;
  z-index: 1;
}
/* class for the output area when it should be height-limited */
div.output_scroll {
  /* ideally, this would be max-height, but FF barfs all over that */
  height: 24em;
  /* FF needs this *and the wrapper* to specify full width, or it will shrinkwrap */
  width: 100%;
  overflow: auto;
  border-radius: 2px;
  -webkit-box-shadow: inset 0 2px 8px rgba(0, 0, 0, 0.8);
  box-shadow: inset 0 2px 8px rgba(0, 0, 0, 0.8);
  display: block;
}
/* output div while it is collapsed */
div.output_collapsed {
  margin: 0px;
  padding: 0px;
  /* Old browsers */
  display: -webkit-box;
  -webkit-box-orient: vertical;
  -webkit-box-align: stretch;
  display: -moz-box;
  -moz-box-orient: vertical;
  -moz-box-align: stretch;
  display: box;
  box-orient: vertical;
  box-align: stretch;
  /* Modern browsers */
  display: flex;
  flex-direction: column;
  align-items: stretch;
}
div.out_prompt_overlay {
  height: 100%;
  padding: 0px 0.4em;
  position: absolute;
  border-radius: 2px;
}
div.out_prompt_overlay:hover {
  /* use inner shadow to get border that is computed the same on WebKit/FF */
  -webkit-box-shadow: inset 0 0 1px #000;
  box-shadow: inset 0 0 1px #000;
  background: rgba(240, 240, 240, 0.5);
}
div.output_prompt {
  color: #D84315;
}
/* This class is the outer container of all output sections. */
div.output_area {
  padding: 0px;
  page-break-inside: avoid;
  /* Old browsers */
  display: -webkit-box;
  -webkit-box-orient: horizontal;
  -webkit-box-align: stretch;
  display: -moz-box;
  -moz-box-orient: horizontal;
  -moz-box-align: stretch;
  display: box;
  box-orient: horizontal;
  box-align: stretch;
  /* Modern browsers */
  display: flex;
  flex-direction: row;
  align-items: stretch;
}
div.output_area .MathJax_Display {
  text-align: left !important;
}
div.output_area .rendered_html table {
  margin-left: 0;
  margin-right: 0;
}
div.output_area .rendered_html img {
  margin-left: 0;
  margin-right: 0;
}
div.output_area img,
div.output_area svg {
  max-width: 100%;
  height: auto;
}
div.output_area img.unconfined,
div.output_area svg.unconfined {
  max-width: none;
}
div.output_area .mglyph > img {
  max-width: none;
}
/* This is needed to protect the pre formating from global settings such
   as that of bootstrap */
.output {
  /* Old browsers */
  display: -webkit-box;
  -webkit-box-orient: vertical;
  -webkit-box-align: stretch;
  display: -moz-box;
  -moz-box-orient: vertical;
  -moz-box-align: stretch;
  display: box;
  box-orient: vertical;
  box-align: stretch;
  /* Modern browsers */
  display: flex;
  flex-direction: column;
  align-items: stretch;
}
@media (max-width: 540px) {
  div.output_area {
    /* Old browsers */
    display: -webkit-box;
    -webkit-box-orient: vertical;
    -webkit-box-align: stretch;
    display: -moz-box;
    -moz-box-orient: vertical;
    -moz-box-align: stretch;
    display: box;
    box-orient: vertical;
    box-align: stretch;
    /* Modern browsers */
    display: flex;
    flex-direction: column;
    align-items: stretch;
  }
}
div.output_area pre {
  margin: 0;
  padding: 1px 0 1px 0;
  border: 0;
  vertical-align: baseline;
  color: black;
  background-color: transparent;
  border-radius: 0;
}
/* This class is for the output subarea inside the output_area and after
   the prompt div. */
div.output_subarea {
  overflow-x: auto;
  padding: 0.4em;
  /* Old browsers */
  -webkit-box-flex: 1;
  -moz-box-flex: 1;
  box-flex: 1;
  /* Modern browsers */
  flex: 1;
  max-width: calc(100% - 14ex);
}
div.output_scroll div.output_subarea {
  overflow-x: visible;
}
/* The rest of the output_* classes are for special styling of the different
   output types */
/* all text output has this class: */
div.output_text {
  text-align: left;
  color: #000;
  /* This has to match that of the the CodeMirror class line-height below */
  line-height: 1.21429em;
}
/* stdout/stderr are 'text' as well as 'stream', but execute_result/error are *not* streams */
div.output_stderr {
  background: #fdd;
  /* very light red background for stderr */
}
div.output_latex {
  text-align: left;
}
/* Empty output_javascript divs should have no height */
div.output_javascript:empty {
  padding: 0;
}
.js-error {
  color: darkred;
}
/* raw_input styles */
div.raw_input_container {
  line-height: 1.21429em;
  padding-top: 5px;
}
pre.raw_input_prompt {
  /* nothing needed here. */
}
input.raw_input {
  font-family: monospace;
  font-size: inherit;
  color: inherit;
  width: auto;
  /* make sure input baseline aligns with prompt */
  vertical-align: baseline;
  /* padding + margin = 0.5em between prompt and cursor */
  padding: 0em 0.25em;
  margin: 0em 0.25em;
}
input.raw_input:focus {
  box-shadow: none;
}
p.p-space {
  margin-bottom: 10px;
}
div.output_unrecognized {
  padding: 5px;
  font-weight: bold;
  color: red;
}
div.output_unrecognized a {
  color: inherit;
  text-decoration: none;
}
div.output_unrecognized a:hover {
  color: inherit;
  text-decoration: none;
}
.rendered_html {
  color: #000;
  /* any extras will just be numbers: */
}
.rendered_html em {
  font-style: italic;
}
.rendered_html strong {
  font-weight: bold;
}
.rendered_html u {
  text-decoration: underline;
}
.rendered_html :link {
  text-decoration: underline;
}
.rendered_html :visited {
  text-decoration: underline;
}
.rendered_html h1 {
  font-size: 185.7%;
  margin: 1.08em 0 0 0;
  font-weight: bold;
  line-height: 1.0;
}
.rendered_html h2 {
  font-size: 157.1%;
  margin: 1.27em 0 0 0;
  font-weight: bold;
  line-height: 1.0;
}
.rendered_html h3 {
  font-size: 128.6%;
  margin: 1.55em 0 0 0;
  font-weight: bold;
  line-height: 1.0;
}
.rendered_html h4 {
  font-size: 100%;
  margin: 2em 0 0 0;
  font-weight: bold;
  line-height: 1.0;
}
.rendered_html h5 {
  font-size: 100%;
  margin: 2em 0 0 0;
  font-weight: bold;
  line-height: 1.0;
  font-style: italic;
}
.rendered_html h6 {
  font-size: 100%;
  margin: 2em 0 0 0;
  font-weight: bold;
  line-height: 1.0;
  font-style: italic;
}
.rendered_html h1:first-child {
  margin-top: 0.538em;
}
.rendered_html h2:first-child {
  margin-top: 0.636em;
}
.rendered_html h3:first-child {
  margin-top: 0.777em;
}
.rendered_html h4:first-child {
  margin-top: 1em;
}
.rendered_html h5:first-child {
  margin-top: 1em;
}
.rendered_html h6:first-child {
  margin-top: 1em;
}
.rendered_html ul:not(.list-inline),
.rendered_html ol:not(.list-inline) {
  padding-left: 2em;
}
.rendered_html ul {
  list-style: disc;
}
.rendered_html ul ul {
  list-style: square;
  margin-top: 0;
}
.rendered_html ul ul ul {
  list-style: circle;
}
.rendered_html ol {
  list-style: decimal;
}
.rendered_html ol ol {
  list-style: upper-alpha;
  margin-top: 0;
}
.rendered_html ol ol ol {
  list-style: lower-alpha;
}
.rendered_html ol ol ol ol {
  list-style: lower-roman;
}
.rendered_html ol ol ol ol ol {
  list-style: decimal;
}
.rendered_html * + ul {
  margin-top: 1em;
}
.rendered_html * + ol {
  margin-top: 1em;
}
.rendered_html hr {
  color: black;
  background-color: black;
}
.rendered_html pre {
  margin: 1em 2em;
  padding: 0px;
  background-color: #fff;
}
.rendered_html code {
  background-color: #eff0f1;
}
.rendered_html p code {
  padding: 1px 5px;
}
.rendered_html pre code {
  background-color: #fff;
}
.rendered_html pre,
.rendered_html code {
  border: 0;
  color: #000;
  font-size: 100%;
}
.rendered_html blockquote {
  margin: 1em 2em;
}
.rendered_html table {
  margin-left: auto;
  margin-right: auto;
  border: none;
  border-collapse: collapse;
  border-spacing: 0;
  color: black;
  font-size: 12px;
  table-layout: fixed;
}
.rendered_html thead {
  border-bottom: 1px solid black;
  vertical-align: bottom;
}
.rendered_html tr,
.rendered_html th,
.rendered_html td {
  text-align: right;
  vertical-align: middle;
  padding: 0.5em 0.5em;
  line-height: normal;
  white-space: normal;
  max-width: none;
  border: none;
}
.rendered_html th {
  font-weight: bold;
}
.rendered_html tbody tr:nth-child(odd) {
  background: #f5f5f5;
}
.rendered_html tbody tr:hover {
  background: rgba(66, 165, 245, 0.2);
}
.rendered_html * + table {
  margin-top: 1em;
}
.rendered_html p {
  text-align: left;
}
.rendered_html * + p {
  margin-top: 1em;
}
.rendered_html img {
  display: block;
  margin-left: auto;
  margin-right: auto;
}
.rendered_html * + img {
  margin-top: 1em;
}
.rendered_html img,
.rendered_html svg {
  max-width: 100%;
  height: auto;
}
.rendered_html img.unconfined,
.rendered_html svg.unconfined {
  max-width: none;
}
.rendered_html .alert {
  margin-bottom: initial;
}
.rendered_html * + .alert {
  margin-top: 1em;
}
[dir="rtl"] .rendered_html p {
  text-align: right;
}
div.text_cell {
  /* Old browsers */
  display: -webkit-box;
  -webkit-box-orient: horizontal;
  -webkit-box-align: stretch;
  display: -moz-box;
  -moz-box-orient: horizontal;
  -moz-box-align: stretch;
  display: box;
  box-orient: horizontal;
  box-align: stretch;
  /* Modern browsers */
  display: flex;
  flex-direction: row;
  align-items: stretch;
}
@media (max-width: 540px) {
  div.text_cell > div.prompt {
    display: none;
  }
}
div.text_cell_render {
  /*font-family: "Helvetica Neue", Arial, Helvetica, Geneva, sans-serif;*/
  outline: none;
  resize: none;
  width: inherit;
  border-style: none;
  padding: 0.5em 0.5em 0.5em 0.4em;
  color: #000;
  box-sizing: border-box;
  -moz-box-sizing: border-box;
  -webkit-box-sizing: border-box;
}
a.anchor-link:link {
  text-decoration: none;
  padding: 0px 20px;
  visibility: hidden;
}
h1:hover .anchor-link,
h2:hover .anchor-link,
h3:hover .anchor-link,
h4:hover .anchor-link,
h5:hover .anchor-link,
h6:hover .anchor-link {
  visibility: visible;
}
.text_cell.rendered .input_area {
  display: none;
}
.text_cell.rendered .rendered_html {
  overflow-x: auto;
  overflow-y: hidden;
}
.text_cell.rendered .rendered_html tr,
.text_cell.rendered .rendered_html th,
.text_cell.rendered .rendered_html td {
  max-width: none;
}
.text_cell.unrendered .text_cell_render {
  display: none;
}
.text_cell .dropzone .input_area {
  border: 2px dashed #bababa;
  margin: -1px;
}
.cm-header-1,
.cm-header-2,
.cm-header-3,
.cm-header-4,
.cm-header-5,
.cm-header-6 {
  font-weight: bold;
  font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
}
.cm-header-1 {
  font-size: 185.7%;
}
.cm-header-2 {
  font-size: 157.1%;
}
.cm-header-3 {
  font-size: 128.6%;
}
.cm-header-4 {
  font-size: 110%;
}
.cm-header-5 {
  font-size: 100%;
  font-style: italic;
}
.cm-header-6 {
  font-size: 100%;
  font-style: italic;
}
/*!
*
* IPython notebook webapp
*
*/
@media (max-width: 767px) {
  .notebook_app {
    padding-left: 0px;
    padding-right: 0px;
  }
}
#ipython-main-app {
  box-sizing: border-box;
  -moz-box-sizing: border-box;
  -webkit-box-sizing: border-box;
  height: 100%;
}
div#notebook_panel {
  margin: 0px;
  padding: 0px;
  box-sizing: border-box;
  -moz-box-sizing: border-box;
  -webkit-box-sizing: border-box;
  height: 100%;
}
div#notebook {
  font-size: 14px;
  line-height: 20px;
  overflow-y: hidden;
  overflow-x: auto;
  width: 100%;
  /* This spaces the page away from the edge of the notebook area */
  padding-top: 20px;
  margin: 0px;
  outline: none;
  box-sizing: border-box;
  -moz-box-sizing: border-box;
  -webkit-box-sizing: border-box;
  min-height: 100%;
}
@media not print {
  #notebook-container {
    padding: 15px;
    background-color: #fff;
    min-height: 0;
    -webkit-box-shadow: 0px 0px 12px 1px rgba(87, 87, 87, 0.2);
    box-shadow: 0px 0px 12px 1px rgba(87, 87, 87, 0.2);
  }
}
@media print {
  #notebook-container {
    width: 100%;
  }
}
div.ui-widget-content {
  border: 1px solid #ababab;
  outline: none;
}
pre.dialog {
  background-color: #f7f7f7;
  border: 1px solid #ddd;
  border-radius: 2px;
  padding: 0.4em;
  padding-left: 2em;
}
p.dialog {
  padding: 0.2em;
}
/* Word-wrap output correctly.  This is the CSS3 spelling, though Firefox seems
   to not honor it correctly.  Webkit browsers (Chrome, rekonq, Safari) do.
 */
pre,
code,
kbd,
samp {
  white-space: pre-wrap;
}
#fonttest {
  font-family: monospace;
}
p {
  margin-bottom: 0;
}
.end_space {
  min-height: 100px;
  transition: height .2s ease;
}
.notebook_app > #header {
  -webkit-box-shadow: 0px 0px 12px 1px rgba(87, 87, 87, 0.2);
  box-shadow: 0px 0px 12px 1px rgba(87, 87, 87, 0.2);
}
@media not print {
  .notebook_app {
    background-color: #EEE;
  }
}
kbd {
  border-style: solid;
  border-width: 1px;
  box-shadow: none;
  margin: 2px;
  padding-left: 2px;
  padding-right: 2px;
  padding-top: 1px;
  padding-bottom: 1px;
}
.jupyter-keybindings {
  padding: 1px;
  line-height: 24px;
  border-bottom: 1px solid gray;
}
.jupyter-keybindings input {
  margin: 0;
  padding: 0;
  border: none;
}
.jupyter-keybindings i {
  padding: 6px;
}
.well code {
  background-color: #ffffff;
  border-color: #ababab;
  border-width: 1px;
  border-style: solid;
  padding: 2px;
  padding-top: 1px;
  padding-bottom: 1px;
}
/* CSS for the cell toolbar */
.celltoolbar {
  border: thin solid #CFCFCF;
  border-bottom: none;
  background: #EEE;
  border-radius: 2px 2px 0px 0px;
  width: 100%;
  height: 29px;
  padding-right: 4px;
  /* Old browsers */
  display: -webkit-box;
  -webkit-box-orient: horizontal;
  -webkit-box-align: stretch;
  display: -moz-box;
  -moz-box-orient: horizontal;
  -moz-box-align: stretch;
  display: box;
  box-orient: horizontal;
  box-align: stretch;
  /* Modern browsers */
  display: flex;
  flex-direction: row;
  align-items: stretch;
  /* Old browsers */
  -webkit-box-pack: end;
  -moz-box-pack: end;
  box-pack: end;
  /* Modern browsers */
  justify-content: flex-end;
  display: -webkit-flex;
}
@media print {
  .celltoolbar {
    display: none;
  }
}
.ctb_hideshow {
  display: none;
  vertical-align: bottom;
}
/* ctb_show is added to the ctb_hideshow div to show the cell toolbar.
   Cell toolbars are only shown when the ctb_global_show class is also set.
*/
.ctb_global_show .ctb_show.ctb_hideshow {
  display: block;
}
.ctb_global_show .ctb_show + .input_area,
.ctb_global_show .ctb_show + div.text_cell_input,
.ctb_global_show .ctb_show ~ div.text_cell_render {
  border-top-right-radius: 0px;
  border-top-left-radius: 0px;
}
.ctb_global_show .ctb_show ~ div.text_cell_render {
  border: 1px solid #cfcfcf;
}
.celltoolbar {
  font-size: 87%;
  padding-top: 3px;
}
.celltoolbar select {
  display: block;
  width: 100%;
  height: 32px;
  padding: 6px 12px;
  font-size: 13px;
  line-height: 1.42857143;
  color: #555555;
  background-color: #fff;
  background-image: none;
  border: 1px solid #ccc;
  border-radius: 2px;
  -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
  box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
  -webkit-transition: border-color ease-in-out .15s, box-shadow ease-in-out .15s;
  -o-transition: border-color ease-in-out .15s, box-shadow ease-in-out .15s;
  transition: border-color ease-in-out .15s, box-shadow ease-in-out .15s;
  height: 30px;
  padding: 5px 10px;
  font-size: 12px;
  line-height: 1.5;
  border-radius: 1px;
  width: inherit;
  font-size: inherit;
  height: 22px;
  padding: 0px;
  display: inline-block;
}
.celltoolbar select:focus {
  border-color: #66afe9;
  outline: 0;
  -webkit-box-shadow: inset 0 1px 1px rgba(0,0,0,.075), 0 0 8px rgba(102, 175, 233, 0.6);
  box-shadow: inset 0 1px 1px rgba(0,0,0,.075), 0 0 8px rgba(102, 175, 233, 0.6);
}
.celltoolbar select::-moz-placeholder {
  color: #999;
  opacity: 1;
}
.celltoolbar select:-ms-input-placeholder {
  color: #999;
}
.celltoolbar select::-webkit-input-placeholder {
  color: #999;
}
.celltoolbar select::-ms-expand {
  border: 0;
  background-color: transparent;
}
.celltoolbar select[disabled],
.celltoolbar select[readonly],
fieldset[disabled] .celltoolbar select {
  background-color: #eeeeee;
  opacity: 1;
}
.celltoolbar select[disabled],
fieldset[disabled] .celltoolbar select {
  cursor: not-allowed;
}
textarea.celltoolbar select {
  height: auto;
}
select.celltoolbar select {
  height: 30px;
  line-height: 30px;
}
textarea.celltoolbar select,
select[multiple].celltoolbar select {
  height: auto;
}
.celltoolbar label {
  margin-left: 5px;
  margin-right: 5px;
}
.tags_button_container {
  width: 100%;
  display: flex;
}
.tag-container {
  display: flex;
  flex-direction: row;
  flex-grow: 1;
  overflow: hidden;
  position: relative;
}
.tag-container > * {
  margin: 0 4px;
}
.remove-tag-btn {
  margin-left: 4px;
}
.tags-input {
  display: flex;
}
.cell-tag:last-child:after {
  content: "";
  position: absolute;
  right: 0;
  width: 40px;
  height: 100%;
  /* Fade to background color of cell toolbar */
  background: linear-gradient(to right, rgba(0, 0, 0, 0), #EEE);
}
.tags-input > * {
  margin-left: 4px;
}
.cell-tag,
.tags-input input,
.tags-input button {
  display: block;
  width: 100%;
  height: 32px;
  padding: 6px 12px;
  font-size: 13px;
  line-height: 1.42857143;
  color: #555555;
  background-color: #fff;
  background-image: none;
  border: 1px solid #ccc;
  border-radius: 2px;
  -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
  box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
  -webkit-transition: border-color ease-in-out .15s, box-shadow ease-in-out .15s;
  -o-transition: border-color ease-in-out .15s, box-shadow ease-in-out .15s;
  transition: border-color ease-in-out .15s, box-shadow ease-in-out .15s;
  height: 30px;
  padding: 5px 10px;
  font-size: 12px;
  line-height: 1.5;
  border-radius: 1px;
  box-shadow: none;
  width: inherit;
  font-size: inherit;
  height: 22px;
  line-height: 22px;
  padding: 0px 4px;
  display: inline-block;
}
.cell-tag:focus,
.tags-input input:focus,
.tags-input button:focus {
  border-color: #66afe9;
  outline: 0;
  -webkit-box-shadow: inset 0 1px 1px rgba(0,0,0,.075), 0 0 8px rgba(102, 175, 233, 0.6);
  box-shadow: inset 0 1px 1px rgba(0,0,0,.075), 0 0 8px rgba(102, 175, 233, 0.6);
}
.cell-tag::-moz-placeholder,
.tags-input input::-moz-placeholder,
.tags-input button::-moz-placeholder {
  color: #999;
  opacity: 1;
}
.cell-tag:-ms-input-placeholder,
.tags-input input:-ms-input-placeholder,
.tags-input button:-ms-input-placeholder {
  color: #999;
}
.cell-tag::-webkit-input-placeholder,
.tags-input input::-webkit-input-placeholder,
.tags-input button::-webkit-input-placeholder {
  color: #999;
}
.cell-tag::-ms-expand,
.tags-input input::-ms-expand,
.tags-input button::-ms-expand {
  border: 0;
  background-color: transparent;
}
.cell-tag[disabled],
.tags-input input[disabled],
.tags-input button[disabled],
.cell-tag[readonly],
.tags-input input[readonly],
.tags-input button[readonly],
fieldset[disabled] .cell-tag,
fieldset[disabled] .tags-input input,
fieldset[disabled] .tags-input button {
  background-color: #eeeeee;
  opacity: 1;
}
.cell-tag[disabled],
.tags-input input[disabled],
.tags-input button[disabled],
fieldset[disabled] .cell-tag,
fieldset[disabled] .tags-input input,
fieldset[disabled] .tags-input button {
  cursor: not-allowed;
}
textarea.cell-tag,
textarea.tags-input input,
textarea.tags-input button {
  height: auto;
}
select.cell-tag,
select.tags-input input,
select.tags-input button {
  height: 30px;
  line-height: 30px;
}
textarea.cell-tag,
textarea.tags-input input,
textarea.tags-input button,
select[multiple].cell-tag,
select[multiple].tags-input input,
select[multiple].tags-input button {
  height: auto;
}
.cell-tag,
.tags-input button {
  padding: 0px 4px;
}
.cell-tag {
  background-color: #fff;
  white-space: nowrap;
}
.tags-input input[type=text]:focus {
  outline: none;
  box-shadow: none;
  border-color: #ccc;
}
.completions {
  position: absolute;
  z-index: 110;
  overflow: hidden;
  border: 1px solid #ababab;
  border-radius: 2px;
  -webkit-box-shadow: 0px 6px 10px -1px #adadad;
  box-shadow: 0px 6px 10px -1px #adadad;
  line-height: 1;
}
.completions select {
  background: white;
  outline: none;
  border: none;
  padding: 0px;
  margin: 0px;
  overflow: auto;
  font-family: monospace;
  font-size: 110%;
  color: #000;
  width: auto;
}
.completions select option.context {
  color: #286090;
}
#kernel_logo_widget .current_kernel_logo {
  display: none;
  margin-top: -1px;
  margin-bottom: -1px;
  width: 32px;
  height: 32px;
}
[dir="rtl"] #kernel_logo_widget {
  float: left !important;
  float: left;
}
.modal .modal-body .move-path {
  display: flex;
  flex-direction: row;
  justify-content: space;
  align-items: center;
}
.modal .modal-body .move-path .server-root {
  padding-right: 20px;
}
.modal .modal-body .move-path .path-input {
  flex: 1;
}
#menubar {
  box-sizing: border-box;
  -moz-box-sizing: border-box;
  -webkit-box-sizing: border-box;
  margin-top: 1px;
}
#menubar .navbar {
  border-top: 1px;
  border-radius: 0px 0px 2px 2px;
  margin-bottom: 0px;
}
#menubar .navbar-toggle {
  float: left;
  padding-top: 7px;
  padding-bottom: 7px;
  border: none;
}
#menubar .navbar-collapse {
  clear: left;
}
[dir="rtl"] #menubar .navbar-toggle {
  float: right;
}
[dir="rtl"] #menubar .navbar-collapse {
  clear: right;
}
[dir="rtl"] #menubar .navbar-nav {
  float: right;
}
[dir="rtl"] #menubar .nav {
  padding-right: 0px;
}
[dir="rtl"] #menubar .navbar-nav > li {
  float: right;
}
[dir="rtl"] #menubar .navbar-right {
  float: left !important;
}
[dir="rtl"] ul.dropdown-menu {
  text-align: right;
  left: auto;
}
[dir="rtl"] ul#new-menu.dropdown-menu {
  right: auto;
  left: 0;
}
.nav-wrapper {
  border-bottom: 1px solid #e7e7e7;
}
i.menu-icon {
  padding-top: 4px;
}
[dir="rtl"] i.menu-icon.pull-right {
  float: left !important;
  float: left;
}
ul#help_menu li a {
  overflow: hidden;
  padding-right: 2.2em;
}
ul#help_menu li a i {
  margin-right: -1.2em;
}
[dir="rtl"] ul#help_menu li a {
  padding-left: 2.2em;
}
[dir="rtl"] ul#help_menu li a i {
  margin-right: 0;
  margin-left: -1.2em;
}
[dir="rtl"] ul#help_menu li a i.pull-right {
  float: left !important;
  float: left;
}
.dropdown-submenu {
  position: relative;
}
.dropdown-submenu > .dropdown-menu {
  top: 0;
  left: 100%;
  margin-top: -6px;
  margin-left: -1px;
}
[dir="rtl"] .dropdown-submenu > .dropdown-menu {
  right: 100%;
  margin-right: -1px;
}
.dropdown-submenu:hover > .dropdown-menu {
  display: block;
}
.dropdown-submenu > a:after {
  display: inline-block;
  font: normal normal normal 14px/1 FontAwesome;
  font-size: inherit;
  text-rendering: auto;
  -webkit-font-smoothing: antialiased;
  -moz-osx-font-smoothing: grayscale;
  display: block;
  content: "\f0da";
  float: right;
  color: #333333;
  margin-top: 2px;
  margin-right: -10px;
}
.dropdown-submenu > a:after.fa-pull-left {
  margin-right: .3em;
}
.dropdown-submenu > a:after.fa-pull-right {
  margin-left: .3em;
}
.dropdown-submenu > a:after.pull-left {
  margin-right: .3em;
}
.dropdown-submenu > a:after.pull-right {
  margin-left: .3em;
}
[dir="rtl"] .dropdown-submenu > a:after {
  float: left;
  content: "\f0d9";
  margin-right: 0;
  margin-left: -10px;
}
.dropdown-submenu:hover > a:after {
  color: #262626;
}
.dropdown-submenu.pull-left {
  float: none;
}
.dropdown-submenu.pull-left > .dropdown-menu {
  left: -100%;
  margin-left: 10px;
}
#notification_area {
  float: right !important;
  float: right;
  z-index: 10;
}
[dir="rtl"] #notification_area {
  float: left !important;
  float: left;
}
.indicator_area {
  float: right !important;
  float: right;
  color: #777;
  margin-left: 5px;
  margin-right: 5px;
  width: 11px;
  z-index: 10;
  text-align: center;
  width: auto;
}
[dir="rtl"] .indicator_area {
  float: left !important;
  float: left;
}
#kernel_indicator {
  float: right !important;
  float: right;
  color: #777;
  margin-left: 5px;
  margin-right: 5px;
  width: 11px;
  z-index: 10;
  text-align: center;
  width: auto;
  border-left: 1px solid;
}
#kernel_indicator .kernel_indicator_name {
  padding-left: 5px;
  padding-right: 5px;
}
[dir="rtl"] #kernel_indicator {
  float: left !important;
  float: left;
  border-left: 0;
  border-right: 1px solid;
}
#modal_indicator {
  float: right !important;
  float: right;
  color: #777;
  margin-left: 5px;
  margin-right: 5px;
  width: 11px;
  z-index: 10;
  text-align: center;
  width: auto;
}
[dir="rtl"] #modal_indicator {
  float: left !important;
  float: left;
}
#readonly-indicator {
  float: right !important;
  float: right;
  color: #777;
  margin-left: 5px;
  margin-right: 5px;
  width: 11px;
  z-index: 10;
  text-align: center;
  width: auto;
  margin-top: 2px;
  margin-bottom: 0px;
  margin-left: 0px;
  margin-right: 0px;
  display: none;
}
.modal_indicator:before {
  width: 1.28571429em;
  text-align: center;
}
.edit_mode .modal_indicator:before {
  display: inline-block;
  font: normal normal normal 14px/1 FontAwesome;
  font-size: inherit;
  text-rendering: auto;
  -webkit-font-smoothing: antialiased;
  -moz-osx-font-smoothing: grayscale;
  content: "\f040";
}
.edit_mode .modal_indicator:before.fa-pull-left {
  margin-right: .3em;
}
.edit_mode .modal_indicator:before.fa-pull-right {
  margin-left: .3em;
}
.edit_mode .modal_indicator:before.pull-left {
  margin-right: .3em;
}
.edit_mode .modal_indicator:before.pull-right {
  margin-left: .3em;
}
.command_mode .modal_indicator:before {
  display: inline-block;
  font: normal normal normal 14px/1 FontAwesome;
  font-size: inherit;
  text-rendering: auto;
  -webkit-font-smoothing: antialiased;
  -moz-osx-font-smoothing: grayscale;
  content: ' ';
}
.command_mode .modal_indicator:before.fa-pull-left {
  margin-right: .3em;
}
.command_mode .modal_indicator:before.fa-pull-right {
  margin-left: .3em;
}
.command_mode .modal_indicator:before.pull-left {
  margin-right: .3em;
}
.command_mode .modal_indicator:before.pull-right {
  margin-left: .3em;
}
.kernel_idle_icon:before {
  display: inline-block;
  font: normal normal normal 14px/1 FontAwesome;
  font-size: inherit;
  text-rendering: auto;
  -webkit-font-smoothing: antialiased;
  -moz-osx-font-smoothing: grayscale;
  content: "\f10c";
}
.kernel_idle_icon:before.fa-pull-left {
  margin-right: .3em;
}
.kernel_idle_icon:before.fa-pull-right {
  margin-left: .3em;
}
.kernel_idle_icon:before.pull-left {
  margin-right: .3em;
}
.kernel_idle_icon:before.pull-right {
  margin-left: .3em;
}
.kernel_busy_icon:before {
  display: inline-block;
  font: normal normal normal 14px/1 FontAwesome;
  font-size: inherit;
  text-rendering: auto;
  -webkit-font-smoothing: antialiased;
  -moz-osx-font-smoothing: grayscale;
  content: "\f111";
}
.kernel_busy_icon:before.fa-pull-left {
  margin-right: .3em;
}
.kernel_busy_icon:before.fa-pull-right {
  margin-left: .3em;
}
.kernel_busy_icon:before.pull-left {
  margin-right: .3em;
}
.kernel_busy_icon:before.pull-right {
  margin-left: .3em;
}
.kernel_dead_icon:before {
  display: inline-block;
  font: normal normal normal 14px/1 FontAwesome;
  font-size: inherit;
  text-rendering: auto;
  -webkit-font-smoothing: antialiased;
  -moz-osx-font-smoothing: grayscale;
  content: "\f1e2";
}
.kernel_dead_icon:before.fa-pull-left {
  margin-right: .3em;
}
.kernel_dead_icon:before.fa-pull-right {
  margin-left: .3em;
}
.kernel_dead_icon:before.pull-left {
  margin-right: .3em;
}
.kernel_dead_icon:before.pull-right {
  margin-left: .3em;
}
.kernel_disconnected_icon:before {
  display: inline-block;
  font: normal normal normal 14px/1 FontAwesome;
  font-size: inherit;
  text-rendering: auto;
  -webkit-font-smoothing: antialiased;
  -moz-osx-font-smoothing: grayscale;
  content: "\f127";
}
.kernel_disconnected_icon:before.fa-pull-left {
  margin-right: .3em;
}
.kernel_disconnected_icon:before.fa-pull-right {
  margin-left: .3em;
}
.kernel_disconnected_icon:before.pull-left {
  margin-right: .3em;
}
.kernel_disconnected_icon:before.pull-right {
  margin-left: .3em;
}
.notification_widget {
  color: #777;
  z-index: 10;
  background: rgba(240, 240, 240, 0.5);
  margin-right: 4px;
  color: #333;
  background-color: #fff;
  border-color: #ccc;
}
.notification_widget:focus,
.notification_widget.focus {
  color: #333;
  background-color: #e6e6e6;
  border-color: #8c8c8c;
}
.notification_widget:hover {
  color: #333;
  background-color: #e6e6e6;
  border-color: #adadad;
}
.notification_widget:active,
.notification_widget.active,
.open > .dropdown-toggle.notification_widget {
  color: #333;
  background-color: #e6e6e6;
  border-color: #adadad;
}
.notification_widget:active:hover,
.notification_widget.active:hover,
.open > .dropdown-toggle.notification_widget:hover,
.notification_widget:active:focus,
.notification_widget.active:focus,
.open > .dropdown-toggle.notification_widget:focus,
.notification_widget:active.focus,
.notification_widget.active.focus,
.open > .dropdown-toggle.notification_widget.focus {
  color: #333;
  background-color: #d4d4d4;
  border-color: #8c8c8c;
}
.notification_widget:active,
.notification_widget.active,
.open > .dropdown-toggle.notification_widget {
  background-image: none;
}
.notification_widget.disabled:hover,
.notification_widget[disabled]:hover,
fieldset[disabled] .notification_widget:hover,
.notification_widget.disabled:focus,
.notification_widget[disabled]:focus,
fieldset[disabled] .notification_widget:focus,
.notification_widget.disabled.focus,
.notification_widget[disabled].focus,
fieldset[disabled] .notification_widget.focus {
  background-color: #fff;
  border-color: #ccc;
}
.notification_widget .badge {
  color: #fff;
  background-color: #333;
}
.notification_widget.warning {
  color: #fff;
  background-color: #f0ad4e;
  border-color: #eea236;
}
.notification_widget.warning:focus,
.notification_widget.warning.focus {
  color: #fff;
  background-color: #ec971f;
  border-color: #985f0d;
}
.notification_widget.warning:hover {
  color: #fff;
  background-color: #ec971f;
  border-color: #d58512;
}
.notification_widget.warning:active,
.notification_widget.warning.active,
.open > .dropdown-toggle.notification_widget.warning {
  color: #fff;
  background-color: #ec971f;
  border-color: #d58512;
}
.notification_widget.warning:active:hover,
.notification_widget.warning.active:hover,
.open > .dropdown-toggle.notification_widget.warning:hover,
.notification_widget.warning:active:focus,
.notification_widget.warning.active:focus,
.open > .dropdown-toggle.notification_widget.warning:focus,
.notification_widget.warning:active.focus,
.notification_widget.warning.active.focus,
.open > .dropdown-toggle.notification_widget.warning.focus {
  color: #fff;
  background-color: #d58512;
  border-color: #985f0d;
}
.notification_widget.warning:active,
.notification_widget.warning.active,
.open > .dropdown-toggle.notification_widget.warning {
  background-image: none;
}
.notification_widget.warning.disabled:hover,
.notification_widget.warning[disabled]:hover,
fieldset[disabled] .notification_widget.warning:hover,
.notification_widget.warning.disabled:focus,
.notification_widget.warning[disabled]:focus,
fieldset[disabled] .notification_widget.warning:focus,
.notification_widget.warning.disabled.focus,
.notification_widget.warning[disabled].focus,
fieldset[disabled] .notification_widget.warning.focus {
  background-color: #f0ad4e;
  border-color: #eea236;
}
.notification_widget.warning .badge {
  color: #f0ad4e;
  background-color: #fff;
}
.notification_widget.success {
  color: #fff;
  background-color: #5cb85c;
  border-color: #4cae4c;
}
.notification_widget.success:focus,
.notification_widget.success.focus {
  color: #fff;
  background-color: #449d44;
  border-color: #255625;
}
.notification_widget.success:hover {
  color: #fff;
  background-color: #449d44;
  border-color: #398439;
}
.notification_widget.success:active,
.notification_widget.success.active,
.open > .dropdown-toggle.notification_widget.success {
  color: #fff;
  background-color: #449d44;
  border-color: #398439;
}
.notification_widget.success:active:hover,
.notification_widget.success.active:hover,
.open > .dropdown-toggle.notification_widget.success:hover,
.notification_widget.success:active:focus,
.notification_widget.success.active:focus,
.open > .dropdown-toggle.notification_widget.success:focus,
.notification_widget.success:active.focus,
.notification_widget.success.active.focus,
.open > .dropdown-toggle.notification_widget.success.focus {
  color: #fff;
  background-color: #398439;
  border-color: #255625;
}
.notification_widget.success:active,
.notification_widget.success.active,
.open > .dropdown-toggle.notification_widget.success {
  background-image: none;
}
.notification_widget.success.disabled:hover,
.notification_widget.success[disabled]:hover,
fieldset[disabled] .notification_widget.success:hover,
.notification_widget.success.disabled:focus,
.notification_widget.success[disabled]:focus,
fieldset[disabled] .notification_widget.success:focus,
.notification_widget.success.disabled.focus,
.notification_widget.success[disabled].focus,
fieldset[disabled] .notification_widget.success.focus {
  background-color: #5cb85c;
  border-color: #4cae4c;
}
.notification_widget.success .badge {
  color: #5cb85c;
  background-color: #fff;
}
.notification_widget.info {
  color: #fff;
  background-color: #5bc0de;
  border-color: #46b8da;
}
.notification_widget.info:focus,
.notification_widget.info.focus {
  color: #fff;
  background-color: #31b0d5;
  border-color: #1b6d85;
}
.notification_widget.info:hover {
  color: #fff;
  background-color: #31b0d5;
  border-color: #269abc;
}
.notification_widget.info:active,
.notification_widget.info.active,
.open > .dropdown-toggle.notification_widget.info {
  color: #fff;
  background-color: #31b0d5;
  border-color: #269abc;
}
.notification_widget.info:active:hover,
.notification_widget.info.active:hover,
.open > .dropdown-toggle.notification_widget.info:hover,
.notification_widget.info:active:focus,
.notification_widget.info.active:focus,
.open > .dropdown-toggle.notification_widget.info:focus,
.notification_widget.info:active.focus,
.notification_widget.info.active.focus,
.open > .dropdown-toggle.notification_widget.info.focus {
  color: #fff;
  background-color: #269abc;
  border-color: #1b6d85;
}
.notification_widget.info:active,
.notification_widget.info.active,
.open > .dropdown-toggle.notification_widget.info {
  background-image: none;
}
.notification_widget.info.disabled:hover,
.notification_widget.info[disabled]:hover,
fieldset[disabled] .notification_widget.info:hover,
.notification_widget.info.disabled:focus,
.notification_widget.info[disabled]:focus,
fieldset[disabled] .notification_widget.info:focus,
.notification_widget.info.disabled.focus,
.notification_widget.info[disabled].focus,
fieldset[disabled] .notification_widget.info.focus {
  background-color: #5bc0de;
  border-color: #46b8da;
}
.notification_widget.info .badge {
  color: #5bc0de;
  background-color: #fff;
}
.notification_widget.danger {
  color: #fff;
  background-color: #d9534f;
  border-color: #d43f3a;
}
.notification_widget.danger:focus,
.notification_widget.danger.focus {
  color: #fff;
  background-color: #c9302c;
  border-color: #761c19;
}
.notification_widget.danger:hover {
  color: #fff;
  background-color: #c9302c;
  border-color: #ac2925;
}
.notification_widget.danger:active,
.notification_widget.danger.active,
.open > .dropdown-toggle.notification_widget.danger {
  color: #fff;
  background-color: #c9302c;
  border-color: #ac2925;
}
.notification_widget.danger:active:hover,
.notification_widget.danger.active:hover,
.open > .dropdown-toggle.notification_widget.danger:hover,
.notification_widget.danger:active:focus,
.notification_widget.danger.active:focus,
.open > .dropdown-toggle.notification_widget.danger:focus,
.notification_widget.danger:active.focus,
.notification_widget.danger.active.focus,
.open > .dropdown-toggle.notification_widget.danger.focus {
  color: #fff;
  background-color: #ac2925;
  border-color: #761c19;
}
.notification_widget.danger:active,
.notification_widget.danger.active,
.open > .dropdown-toggle.notification_widget.danger {
  background-image: none;
}
.notification_widget.danger.disabled:hover,
.notification_widget.danger[disabled]:hover,
fieldset[disabled] .notification_widget.danger:hover,
.notification_widget.danger.disabled:focus,
.notification_widget.danger[disabled]:focus,
fieldset[disabled] .notification_widget.danger:focus,
.notification_widget.danger.disabled.focus,
.notification_widget.danger[disabled].focus,
fieldset[disabled] .notification_widget.danger.focus {
  background-color: #d9534f;
  border-color: #d43f3a;
}
.notification_widget.danger .badge {
  color: #d9534f;
  background-color: #fff;
}
div#pager {
  background-color: #fff;
  font-size: 14px;
  line-height: 20px;
  overflow: hidden;
  display: none;
  position: fixed;
  bottom: 0px;
  width: 100%;
  max-height: 50%;
  padding-top: 8px;
  -webkit-box-shadow: 0px 0px 12px 1px rgba(87, 87, 87, 0.2);
  box-shadow: 0px 0px 12px 1px rgba(87, 87, 87, 0.2);
  /* Display over codemirror */
  z-index: 100;
  /* Hack which prevents jquery ui resizable from changing top. */
  top: auto !important;
}
div#pager pre {
  line-height: 1.21429em;
  color: #000;
  background-color: #f7f7f7;
  padding: 0.4em;
}
div#pager #pager-button-area {
  position: absolute;
  top: 8px;
  right: 20px;
}
div#pager #pager-contents {
  position: relative;
  overflow: auto;
  width: 100%;
  height: 100%;
}
div#pager #pager-contents #pager-container {
  position: relative;
  padding: 15px 0px;
  box-sizing: border-box;
  -moz-box-sizing: border-box;
  -webkit-box-sizing: border-box;
}
div#pager .ui-resizable-handle {
  top: 0px;
  height: 8px;
  background: #f7f7f7;
  border-top: 1px solid #cfcfcf;
  border-bottom: 1px solid #cfcfcf;
  /* This injects handle bars (a short, wide = symbol) for 
        the resize handle. */
}
div#pager .ui-resizable-handle::after {
  content: '';
  top: 2px;
  left: 50%;
  height: 3px;
  width: 30px;
  margin-left: -15px;
  position: absolute;
  border-top: 1px solid #cfcfcf;
}
.quickhelp {
  /* Old browsers */
  display: -webkit-box;
  -webkit-box-orient: horizontal;
  -webkit-box-align: stretch;
  display: -moz-box;
  -moz-box-orient: horizontal;
  -moz-box-align: stretch;
  display: box;
  box-orient: horizontal;
  box-align: stretch;
  /* Modern browsers */
  display: flex;
  flex-direction: row;
  align-items: stretch;
  line-height: 1.8em;
}
.shortcut_key {
  display: inline-block;
  width: 21ex;
  text-align: right;
  font-family: monospace;
}
.shortcut_descr {
  display: inline-block;
  /* Old browsers */
  -webkit-box-flex: 1;
  -moz-box-flex: 1;
  box-flex: 1;
  /* Modern browsers */
  flex: 1;
}
span.save_widget {
  height: 30px;
  margin-top: 4px;
  display: flex;
  justify-content: flex-start;
  align-items: baseline;
  width: 50%;
  flex: 1;
}
span.save_widget span.filename {
  height: 100%;
  line-height: 1em;
  margin-left: 16px;
  border: none;
  font-size: 146.5%;
  text-overflow: ellipsis;
  overflow: hidden;
  white-space: nowrap;
  border-radius: 2px;
}
span.save_widget span.filename:hover {
  background-color: #e6e6e6;
}
[dir="rtl"] span.save_widget.pull-left {
  float: right !important;
  float: right;
}
[dir="rtl"] span.save_widget span.filename {
  margin-left: 0;
  margin-right: 16px;
}
span.checkpoint_status,
span.autosave_status {
  font-size: small;
  white-space: nowrap;
  padding: 0 5px;
}
@media (max-width: 767px) {
  span.save_widget {
    font-size: small;
    padding: 0 0 0 5px;
  }
  span.checkpoint_status,
  span.autosave_status {
    display: none;
  }
}
@media (min-width: 768px) and (max-width: 991px) {
  span.checkpoint_status {
    display: none;
  }
  span.autosave_status {
    font-size: x-small;
  }
}
.toolbar {
  padding: 0px;
  margin-left: -5px;
  margin-top: 2px;
  margin-bottom: 5px;
  box-sizing: border-box;
  -moz-box-sizing: border-box;
  -webkit-box-sizing: border-box;
}
.toolbar select,
.toolbar label {
  width: auto;
  vertical-align: middle;
  margin-right: 2px;
  margin-bottom: 0px;
  display: inline;
  font-size: 92%;
  margin-left: 0.3em;
  margin-right: 0.3em;
  padding: 0px;
  padding-top: 3px;
}
.toolbar .btn {
  padding: 2px 8px;
}
.toolbar .btn-group {
  margin-top: 0px;
  margin-left: 5px;
}
.toolbar-btn-label {
  margin-left: 6px;
}
#maintoolbar {
  margin-bottom: -3px;
  margin-top: -8px;
  border: 0px;
  min-height: 27px;
  margin-left: 0px;
  padding-top: 11px;
  padding-bottom: 3px;
}
#maintoolbar .navbar-text {
  float: none;
  vertical-align: middle;
  text-align: right;
  margin-left: 5px;
  margin-right: 0px;
  margin-top: 0px;
}
.select-xs {
  height: 24px;
}
[dir="rtl"] .btn-group > .btn,
.btn-group-vertical > .btn {
  float: right;
}
.pulse,
.dropdown-menu > li > a.pulse,
li.pulse > a.dropdown-toggle,
li.pulse.open > a.dropdown-toggle {
  background-color: #F37626;
  color: white;
}
/**
 * Primary styles
 *
 * Author: Jupyter Development Team
 */
/** WARNING IF YOU ARE EDITTING THIS FILE, if this is a .css file, It has a lot
 * of chance of beeing generated from the ../less/[samename].less file, you can
 * try to get back the less file by reverting somme commit in history
 **/
/*
 * We'll try to get something pretty, so we
 * have some strange css to have the scroll bar on
 * the left with fix button on the top right of the tooltip
 */
@-moz-keyframes fadeOut {
  from {
    opacity: 1;
  }
  to {
    opacity: 0;
  }
}
@-webkit-keyframes fadeOut {
  from {
    opacity: 1;
  }
  to {
    opacity: 0;
  }
}
@-moz-keyframes fadeIn {
  from {
    opacity: 0;
  }
  to {
    opacity: 1;
  }
}
@-webkit-keyframes fadeIn {
  from {
    opacity: 0;
  }
  to {
    opacity: 1;
  }
}
/*properties of tooltip after "expand"*/
.bigtooltip {
  overflow: auto;
  height: 200px;
  -webkit-transition-property: height;
  -webkit-transition-duration: 500ms;
  -moz-transition-property: height;
  -moz-transition-duration: 500ms;
  transition-property: height;
  transition-duration: 500ms;
}
/*properties of tooltip before "expand"*/
.smalltooltip {
  -webkit-transition-property: height;
  -webkit-transition-duration: 500ms;
  -moz-transition-property: height;
  -moz-transition-duration: 500ms;
  transition-property: height;
  transition-duration: 500ms;
  text-overflow: ellipsis;
  overflow: hidden;
  height: 80px;
}
.tooltipbuttons {
  position: absolute;
  padding-right: 15px;
  top: 0px;
  right: 0px;
}
.tooltiptext {
  /*avoid the button to overlap on some docstring*/
  padding-right: 30px;
}
.ipython_tooltip {
  max-width: 700px;
  /*fade-in animation when inserted*/
  -webkit-animation: fadeOut 400ms;
  -moz-animation: fadeOut 400ms;
  animation: fadeOut 400ms;
  -webkit-animation: fadeIn 400ms;
  -moz-animation: fadeIn 400ms;
  animation: fadeIn 400ms;
  vertical-align: middle;
  background-color: #f7f7f7;
  overflow: visible;
  border: #ababab 1px solid;
  outline: none;
  padding: 3px;
  margin: 0px;
  padding-left: 7px;
  font-family: monospace;
  min-height: 50px;
  -moz-box-shadow: 0px 6px 10px -1px #adadad;
  -webkit-box-shadow: 0px 6px 10px -1px #adadad;
  box-shadow: 0px 6px 10px -1px #adadad;
  border-radius: 2px;
  position: absolute;
  z-index: 1000;
}
.ipython_tooltip a {
  float: right;
}
.ipython_tooltip .tooltiptext pre {
  border: 0;
  border-radius: 0;
  font-size: 100%;
  background-color: #f7f7f7;
}
.pretooltiparrow {
  left: 0px;
  margin: 0px;
  top: -16px;
  width: 40px;
  height: 16px;
  overflow: hidden;
  position: absolute;
}
.pretooltiparrow:before {
  background-color: #f7f7f7;
  border: 1px #ababab solid;
  z-index: 11;
  content: "";
  position: absolute;
  left: 15px;
  top: 10px;
  width: 25px;
  height: 25px;
  -webkit-transform: rotate(45deg);
  -moz-transform: rotate(45deg);
  -ms-transform: rotate(45deg);
  -o-transform: rotate(45deg);
}
ul.typeahead-list i {
  margin-left: -10px;
  width: 18px;
}
[dir="rtl"] ul.typeahead-list i {
  margin-left: 0;
  margin-right: -10px;
}
ul.typeahead-list {
  max-height: 80vh;
  overflow: auto;
}
ul.typeahead-list > li > a {
  /** Firefox bug **/
  /* see https://github.com/jupyter/notebook/issues/559 */
  white-space: normal;
}
ul.typeahead-list  > li > a.pull-right {
  float: left !important;
  float: left;
}
[dir="rtl"] .typeahead-list {
  text-align: right;
}
.cmd-palette .modal-body {
  padding: 7px;
}
.cmd-palette form {
  background: white;
}
.cmd-palette input {
  outline: none;
}
.no-shortcut {
  min-width: 20px;
  color: transparent;
}
[dir="rtl"] .no-shortcut.pull-right {
  float: left !important;
  float: left;
}
[dir="rtl"] .command-shortcut.pull-right {
  float: left !important;
  float: left;
}
.command-shortcut:before {
  content: "(command mode)";
  padding-right: 3px;
  color: #777777;
}
.edit-shortcut:before {
  content: "(edit)";
  padding-right: 3px;
  color: #777777;
}
[dir="rtl"] .edit-shortcut.pull-right {
  float: left !important;
  float: left;
}
#find-and-replace #replace-preview .match,
#find-and-replace #replace-preview .insert {
  background-color: #BBDEFB;
  border-color: #90CAF9;
  border-style: solid;
  border-width: 1px;
  border-radius: 0px;
}
[dir="ltr"] #find-and-replace .input-group-btn + .form-control {
  border-left: none;
}
[dir="rtl"] #find-and-replace .input-group-btn + .form-control {
  border-right: none;
}
#find-and-replace #replace-preview .replace .match {
  background-color: #FFCDD2;
  border-color: #EF9A9A;
  border-radius: 0px;
}
#find-and-replace #replace-preview .replace .insert {
  background-color: #C8E6C9;
  border-color: #A5D6A7;
  border-radius: 0px;
}
#find-and-replace #replace-preview {
  max-height: 60vh;
  overflow: auto;
}
#find-and-replace #replace-preview pre {
  padding: 5px 10px;
}
.terminal-app {
  background: #EEE;
}
.terminal-app #header {
  background: #fff;
  -webkit-box-shadow: 0px 0px 12px 1px rgba(87, 87, 87, 0.2);
  box-shadow: 0px 0px 12px 1px rgba(87, 87, 87, 0.2);
}
.terminal-app .terminal {
  width: 100%;
  float: left;
  font-family: monospace;
  color: white;
  background: black;
  padding: 0.4em;
  border-radius: 2px;
  -webkit-box-shadow: 0px 0px 12px 1px rgba(87, 87, 87, 0.4);
  box-shadow: 0px 0px 12px 1px rgba(87, 87, 87, 0.4);
}
.terminal-app .terminal,
.terminal-app .terminal dummy-screen {
  line-height: 1em;
  font-size: 14px;
}
.terminal-app .terminal .xterm-rows {
  padding: 10px;
}
.terminal-app .terminal-cursor {
  color: black;
  background: white;
}
.terminal-app #terminado-container {
  margin-top: 20px;
}
/*# sourceMappingURL=style.min.css.map */
    </style>
<style type="text/css">
    .highlight .hll { background-color: #ffffcc }
.highlight  { background: #f8f8f8; }
.highlight .c { color: #408080; font-style: italic } /* Comment */
.highlight .err { border: 1px solid #FF0000 } /* Error */
.highlight .k { color: #008000; font-weight: bold } /* Keyword */
.highlight .o { color: #666666 } /* Operator */
.highlight .ch { color: #408080; font-style: italic } /* Comment.Hashbang */
.highlight .cm { color: #408080; font-style: italic } /* Comment.Multiline */
.highlight .cp { color: #BC7A00 } /* Comment.Preproc */
.highlight .cpf { color: #408080; font-style: italic } /* Comment.PreprocFile */
.highlight .c1 { color: #408080; font-style: italic } /* Comment.Single */
.highlight .cs { color: #408080; font-style: italic } /* Comment.Special */
.highlight .gd { color: #A00000 } /* Generic.Deleted */
.highlight .ge { font-style: italic } /* Generic.Emph */
.highlight .gr { color: #FF0000 } /* Generic.Error */
.highlight .gh { color: #000080; font-weight: bold } /* Generic.Heading */
.highlight .gi { color: #00A000 } /* Generic.Inserted */
.highlight .go { color: #888888 } /* Generic.Output */
.highlight .gp { color: #000080; font-weight: bold } /* Generic.Prompt */
.highlight .gs { font-weight: bold } /* Generic.Strong */
.highlight .gu { color: #800080; font-weight: bold } /* Generic.Subheading */
.highlight .gt { color: #0044DD } /* Generic.Traceback */
.highlight .kc { color: #008000; font-weight: bold } /* Keyword.Constant */
.highlight .kd { color: #008000; font-weight: bold } /* Keyword.Declaration */
.highlight .kn { color: #008000; font-weight: bold } /* Keyword.Namespace */
.highlight .kp { color: #008000 } /* Keyword.Pseudo */
.highlight .kr { color: #008000; font-weight: bold } /* Keyword.Reserved */
.highlight .kt { color: #B00040 } /* Keyword.Type */
.highlight .m { color: #666666 } /* Literal.Number */
.highlight .s { color: #BA2121 } /* Literal.String */
.highlight .na { color: #7D9029 } /* Name.Attribute */
.highlight .nb { color: #008000 } /* Name.Builtin */
.highlight .nc { color: #0000FF; font-weight: bold } /* Name.Class */
.highlight .no { color: #880000 } /* Name.Constant */
.highlight .nd { color: #AA22FF } /* Name.Decorator */
.highlight .ni { color: #999999; font-weight: bold } /* Name.Entity */
.highlight .ne { color: #D2413A; font-weight: bold } /* Name.Exception */
.highlight .nf { color: #0000FF } /* Name.Function */
.highlight .nl { color: #A0A000 } /* Name.Label */
.highlight .nn { color: #0000FF; font-weight: bold } /* Name.Namespace */
.highlight .nt { color: #008000; font-weight: bold } /* Name.Tag */
.highlight .nv { color: #19177C } /* Name.Variable */
.highlight .ow { color: #AA22FF; font-weight: bold } /* Operator.Word */
.highlight .w { color: #bbbbbb } /* Text.Whitespace */
.highlight .mb { color: #666666 } /* Literal.Number.Bin */
.highlight .mf { color: #666666 } /* Literal.Number.Float */
.highlight .mh { color: #666666 } /* Literal.Number.Hex */
.highlight .mi { color: #666666 } /* Literal.Number.Integer */
.highlight .mo { color: #666666 } /* Literal.Number.Oct */
.highlight .sa { color: #BA2121 } /* Literal.String.Affix */
.highlight .sb { color: #BA2121 } /* Literal.String.Backtick */
.highlight .sc { color: #BA2121 } /* Literal.String.Char */
.highlight .dl { color: #BA2121 } /* Literal.String.Delimiter */
.highlight .sd { color: #BA2121; font-style: italic } /* Literal.String.Doc */
.highlight .s2 { color: #BA2121 } /* Literal.String.Double */
.highlight .se { color: #BB6622; font-weight: bold } /* Literal.String.Escape */
.highlight .sh { color: #BA2121 } /* Literal.String.Heredoc */
.highlight .si { color: #BB6688; font-weight: bold } /* Literal.String.Interpol */
.highlight .sx { color: #008000 } /* Literal.String.Other */
.highlight .sr { color: #BB6688 } /* Literal.String.Regex */
.highlight .s1 { color: #BA2121 } /* Literal.String.Single */
.highlight .ss { color: #19177C } /* Literal.String.Symbol */
.highlight .bp { color: #008000 } /* Name.Builtin.Pseudo */
.highlight .fm { color: #0000FF } /* Name.Function.Magic */
.highlight .vc { color: #19177C } /* Name.Variable.Class */
.highlight .vg { color: #19177C } /* Name.Variable.Global */
.highlight .vi { color: #19177C } /* Name.Variable.Instance */
.highlight .vm { color: #19177C } /* Name.Variable.Magic */
.highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
    </style>


<style type="text/css">
/* Overrides of notebook CSS for static HTML export */
body {
  overflow: visible;
  padding: 8px;
}

div#notebook {
  overflow: visible;
  border-top: none;
}@media print {
  div.cell {
    display: block;
    page-break-inside: avoid;
  } 
  div.output_wrapper { 
    display: block;
    page-break-inside: avoid; 
  }
  div.output { 
    display: block;
    page-break-inside: avoid; 
  }
}
</style>

<!-- Custom stylesheet, it must be in the same directory as the html file -->
<link rel="stylesheet" href="custom.css">

<!-- Loading mathjax macro -->
<!-- Load mathjax -->
    <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/latest.js?config=TeX-AMS_HTML"></script>
    <!-- MathJax configuration -->
    <script type="text/x-mathjax-config">
    MathJax.Hub.Config({
        tex2jax: {
            inlineMath: [ ['$','$'], ["\\(","\\)"] ],
            displayMath: [ ['$$','$$'], ["\\[","\\]"] ],
            processEscapes: true,
            processEnvironments: true
        },
        // Center justify equations in code and markdown cells. Elsewhere
        // we use CSS to left justify single line equations in code cells.
        displayAlign: 'center',
        "HTML-CSS": {
            styles: {'.MathJax_Display': {"margin": 0}},
            linebreaks: { automatic: true }
        }
    });
    </script>
    <!-- End of mathjax configuration --></head>
<body>
  <div tabindex="-1" id="notebook" class="border-box-sizing">
    <div class="container" id="notebook-container">

<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
</div><div class="inner_cell">
<div class="text_cell_render border-box-sizing rendered_html">
<h1 id="Transformer-&#27169;&#22411;">Transformer &#27169;&#22411;<a class="anchor-link" href="#Transformer-&#27169;&#22411;">&#182;</a></h1>
</div>
</div>
</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="prompt input_prompt">In&nbsp;[1]:</div>
<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="kn">from</span> <span class="nn">google.colab</span> <span class="kn">import</span> <span class="n">drive</span>
<span class="n">drive</span><span class="o">.</span><span class="n">mount</span><span class="p">(</span><span class="s1">&#39;/content/drive&#39;</span><span class="p">)</span>
</pre></div>

    </div>
</div>
</div>

<div class="output_wrapper">
<div class="output">


<div class="output_area">

    <div class="prompt"></div>


<div class="output_subarea output_stream output_stdout output_text">
<pre>Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&amp;redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&amp;response_type=code&amp;scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly

Enter your authorization code:
··········
Mounted at /content/drive
</pre>
</div>
</div>

</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="prompt input_prompt">In&nbsp;[3]:</div>
<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="o">%</span><span class="k">tensorflow_version</span> 2.x
<span class="kn">import</span> <span class="nn">tensorflow</span> <span class="k">as</span> <span class="nn">tf</span>
<span class="nb">print</span><span class="p">(</span><span class="n">tf</span><span class="o">.</span><span class="n">test</span><span class="o">.</span><span class="n">is_gpu_available</span><span class="p">())</span>
</pre></div>

    </div>
</div>
</div>

<div class="output_wrapper">
<div class="output">


<div class="output_area">

    <div class="prompt"></div>


<div class="output_subarea output_stream output_stdout output_text">
<pre>TensorFlow 2.x selected.
WARNING:tensorflow:From &lt;ipython-input-3-b2ae02e03e62&gt;:3: is_gpu_available (from tensorflow.python.framework.test_util) is deprecated and will be removed in a future version.
Instructions for updating:
Use `tf.config.list_physical_devices(&#39;GPU&#39;)` instead.
True
</pre>
</div>
</div>

</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="prompt input_prompt">In&nbsp;[4]:</div>
<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="kn">import</span> <span class="nn">matplotlib</span> <span class="k">as</span> <span class="nn">mpl</span>
<span class="kn">import</span> <span class="nn">matplotlib.pyplot</span> <span class="k">as</span> <span class="nn">plt</span>
<span class="o">%</span><span class="k">matplotlib</span> inline
<span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
<span class="kn">import</span> <span class="nn">sklearn</span>
<span class="kn">import</span> <span class="nn">pandas</span> <span class="k">as</span> <span class="nn">pd</span>
<span class="kn">import</span> <span class="nn">os</span>
<span class="kn">import</span> <span class="nn">sys</span>
<span class="kn">import</span> <span class="nn">time</span>
<span class="kn">import</span> <span class="nn">tensorflow</span> <span class="k">as</span> <span class="nn">tf</span>
<span class="kn">from</span> <span class="nn">tensorflow</span> <span class="kn">import</span> <span class="n">keras</span>

<span class="nb">print</span><span class="p">(</span><span class="n">tf</span><span class="o">.</span><span class="n">__version__</span><span class="p">)</span>
<span class="nb">print</span><span class="p">(</span><span class="n">sys</span><span class="o">.</span><span class="n">version_info</span><span class="p">)</span>
<span class="k">for</span> <span class="n">module</span> <span class="ow">in</span> <span class="n">mpl</span><span class="p">,</span><span class="n">np</span><span class="p">,</span><span class="n">pd</span><span class="p">,</span><span class="n">sklearn</span><span class="p">,</span><span class="n">tf</span><span class="p">,</span><span class="n">keras</span><span class="p">:</span>
    <span class="nb">print</span><span class="p">(</span><span class="n">module</span><span class="o">.</span><span class="vm">__name__</span><span class="p">,</span><span class="n">module</span><span class="o">.</span><span class="n">__version__</span><span class="p">)</span>
</pre></div>

    </div>
</div>
</div>

<div class="output_wrapper">
<div class="output">


<div class="output_area">

    <div class="prompt"></div>


<div class="output_subarea output_stream output_stdout output_text">
<pre>2.1.0
sys.version_info(major=3, minor=6, micro=9, releaselevel=&#39;final&#39;, serial=0)
matplotlib 3.1.3
numpy 1.17.5
pandas 0.25.3
sklearn 0.22.1
tensorflow 2.1.0
tensorflow_core.python.keras.api._v2.keras 2.2.4-tf
</pre>
</div>
</div>

</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="prompt input_prompt">In&nbsp;[&nbsp;]:</div>
<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="c1"># gpus = tf.config.experimental.list_physical_devices(&#39;GPU&#39;)</span>
<span class="c1"># for gpu in gpus:</span>
<span class="c1">#     tf.config.experimental.set_memory_growth(gpu, True)</span>
</pre></div>

    </div>
</div>
</div>

</div>
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
</div><div class="inner_cell">
<div class="text_cell_render border-box-sizing rendered_html">
<p>步骤：</p>
<ol>
<li>loads data</li>
<li>preprocessess data -&gt; dataset</li>
<li>tools<ul>
<li>generates position embedding</li>
<li>create mask (a. padding, b. decoder)</li>
<li>scaled_dot_product_attention</li>
</ul>
</li>
<li>build model<ul>
<li>MultiheadAttention</li>
<li>EncoderLayer</li>
<li>DecoderLayer</li>
<li>EncoderModel</li>
<li>DecoderModel</li>
<li>Transformer</li>
</ul>
</li>
<li>train<ul>
<li>initializes model</li>
<li>define loss, optimizer, learning_rate schedule</li>
<li>train_step</li>
<li>train process</li>
</ul>
</li>
<li>train step -&gt; train process</li>
<li>Evaluate and Visualize</li>
</ol>

</div>
</div>
</div>
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
</div><div class="inner_cell">
<div class="text_cell_render border-box-sizing rendered_html">
<h2 id="&#25968;&#25454;&#22788;&#29702;">&#25968;&#25454;&#22788;&#29702;<a class="anchor-link" href="#&#25968;&#25454;&#22788;&#29702;">&#182;</a></h2><h3 id="&#21152;&#36733;&#25968;&#25454;">&#21152;&#36733;&#25968;&#25454;<a class="anchor-link" href="#&#21152;&#36733;&#25968;&#25454;">&#182;</a></h3>
</div>
</div>
</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="prompt input_prompt">In&nbsp;[6]:</div>
<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="kn">import</span> <span class="nn">tensorflow_datasets</span> <span class="k">as</span> <span class="nn">tfds</span>

<span class="n">examples</span><span class="p">,</span> <span class="n">info</span> <span class="o">=</span> <span class="n">tfds</span><span class="o">.</span><span class="n">load</span><span class="p">(</span><span class="s1">&#39;ted_hrlr_translate/pt_to_en&#39;</span><span class="p">,</span>
                           <span class="n">with_info</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span> <span class="n">as_supervised</span> <span class="o">=</span> <span class="kc">True</span><span class="p">)</span>
<span class="n">train_examples</span><span class="p">,</span> <span class="n">val_examples</span> <span class="o">=</span> <span class="n">examples</span><span class="p">[</span><span class="s1">&#39;train&#39;</span><span class="p">],</span> <span class="n">examples</span><span class="p">[</span><span class="s1">&#39;validation&#39;</span><span class="p">]</span>

<span class="nb">print</span><span class="p">(</span><span class="n">info</span><span class="p">)</span>
</pre></div>

    </div>
</div>
</div>

<div class="output_wrapper">
<div class="output">


<div class="output_area">

    <div class="prompt"></div>


<div class="output_subarea output_stream output_stdout output_text">
<pre><span class="ansi-bold">Downloading and preparing dataset ted_hrlr_translate (124.94 MiB) to /root/tensorflow_datasets/ted_hrlr_translate/pt_to_en/1.0.0...</span>
</pre>
</div>
</div>

<div class="output_area">

    <div class="prompt"></div>





 
 
<div id="d5cfa22f-f091-4406-a162-51cb77c7f8dc"></div>
<div class="output_subarea output_widget_view ">
<script type="text/javascript">
var element = $('#d5cfa22f-f091-4406-a162-51cb77c7f8dc');
</script>
<script type="application/vnd.jupyter.widget-view+json">
{"model_id": "b20f695e03d748bdb2016a09c6712e6a", "version_major": 2, "version_minor": 0}
</script>
</div>

</div>

<div class="output_area">

    <div class="prompt"></div>





 
 
<div id="5c98e0f6-d191-4dab-b69e-c29473242136"></div>
<div class="output_subarea output_widget_view ">
<script type="text/javascript">
var element = $('#5c98e0f6-d191-4dab-b69e-c29473242136');
</script>
<script type="application/vnd.jupyter.widget-view+json">
{"model_id": "ea263382231d42b29888fb3d1b16acca", "version_major": 2, "version_minor": 0}
</script>
</div>

</div>

<div class="output_area">

    <div class="prompt"></div>





 
 
<div id="da53ad44-e121-43c7-9266-5c2cf0d0b6b2"></div>
<div class="output_subarea output_widget_view ">
<script type="text/javascript">
var element = $('#da53ad44-e121-43c7-9266-5c2cf0d0b6b2');
</script>
<script type="application/vnd.jupyter.widget-view+json">
{"model_id": "918b595ca82f4e889e8851e7a3301542", "version_major": 2, "version_minor": 0}
</script>
</div>

</div>

<div class="output_area">

    <div class="prompt"></div>


<div class="output_subarea output_stream output_stdout output_text">
<pre>



</pre>
</div>
</div>

<div class="output_area">

    <div class="prompt"></div>





 
 
<div id="c0b65f45-98a4-4dc4-afcf-b99c054546a3"></div>
<div class="output_subarea output_widget_view ">
<script type="text/javascript">
var element = $('#c0b65f45-98a4-4dc4-afcf-b99c054546a3');
</script>
<script type="application/vnd.jupyter.widget-view+json">
{"model_id": "8e4d74fcb67d49378495b45510ff668c", "version_major": 2, "version_minor": 0}
</script>
</div>

</div>

<div class="output_area">

    <div class="prompt"></div>


<div class="output_subarea output_stream output_stdout output_text">
<pre>Shuffling and writing examples to /root/tensorflow_datasets/ted_hrlr_translate/pt_to_en/1.0.0.incomplete6QGB3C/ted_hrlr_translate-train.tfrecord
</pre>
</div>
</div>

<div class="output_area">

    <div class="prompt"></div>





 
 
<div id="13a67cc6-6fae-4774-be4f-74d4a40a7661"></div>
<div class="output_subarea output_widget_view ">
<script type="text/javascript">
var element = $('#13a67cc6-6fae-4774-be4f-74d4a40a7661');
</script>
<script type="application/vnd.jupyter.widget-view+json">
{"model_id": "c6c4fa9e09454d6f81d9e7cabc1ad9d8", "version_major": 2, "version_minor": 0}
</script>
</div>

</div>

<div class="output_area">

    <div class="prompt"></div>


<div class="output_subarea output_stream output_stdout output_text">
<pre></pre>
</div>
</div>

<div class="output_area">

    <div class="prompt"></div>





 
 
<div id="e2db6b5d-65a9-4444-b646-7ea6fd50fa4e"></div>
<div class="output_subarea output_widget_view ">
<script type="text/javascript">
var element = $('#e2db6b5d-65a9-4444-b646-7ea6fd50fa4e');
</script>
<script type="application/vnd.jupyter.widget-view+json">
{"model_id": "ed49da325e7146d5bddbd6b2ee6e338b", "version_major": 2, "version_minor": 0}
</script>
</div>

</div>

<div class="output_area">

    <div class="prompt"></div>


<div class="output_subarea output_stream output_stdout output_text">
<pre>Shuffling and writing examples to /root/tensorflow_datasets/ted_hrlr_translate/pt_to_en/1.0.0.incomplete6QGB3C/ted_hrlr_translate-validation.tfrecord
</pre>
</div>
</div>

<div class="output_area">

    <div class="prompt"></div>





 
 
<div id="2ba7eb23-206e-4505-9bb4-fa442941cc3b"></div>
<div class="output_subarea output_widget_view ">
<script type="text/javascript">
var element = $('#2ba7eb23-206e-4505-9bb4-fa442941cc3b');
</script>
<script type="application/vnd.jupyter.widget-view+json">
{"model_id": "34171b17684e49708e0633eed1c8216c", "version_major": 2, "version_minor": 0}
</script>
</div>

</div>

<div class="output_area">

    <div class="prompt"></div>


<div class="output_subarea output_stream output_stdout output_text">
<pre></pre>
</div>
</div>

<div class="output_area">

    <div class="prompt"></div>





 
 
<div id="a4d63222-eccd-4cf6-b177-f24290187750"></div>
<div class="output_subarea output_widget_view ">
<script type="text/javascript">
var element = $('#a4d63222-eccd-4cf6-b177-f24290187750');
</script>
<script type="application/vnd.jupyter.widget-view+json">
{"model_id": "8543fc0a11d841bb9593d942eaf05601", "version_major": 2, "version_minor": 0}
</script>
</div>

</div>

<div class="output_area">

    <div class="prompt"></div>


<div class="output_subarea output_stream output_stdout output_text">
<pre>Shuffling and writing examples to /root/tensorflow_datasets/ted_hrlr_translate/pt_to_en/1.0.0.incomplete6QGB3C/ted_hrlr_translate-test.tfrecord
</pre>
</div>
</div>

<div class="output_area">

    <div class="prompt"></div>





 
 
<div id="130de1ec-7a79-4477-9440-9e3c6c65f98a"></div>
<div class="output_subarea output_widget_view ">
<script type="text/javascript">
var element = $('#130de1ec-7a79-4477-9440-9e3c6c65f98a');
</script>
<script type="application/vnd.jupyter.widget-view+json">
{"model_id": "8a66272607e04f92b935580029abe3a1", "version_major": 2, "version_minor": 0}
</script>
</div>

</div>

<div class="output_area">

    <div class="prompt"></div>


<div class="output_subarea output_stream output_stdout output_text">
<pre><span class="ansi-bold">Dataset ted_hrlr_translate downloaded and prepared to /root/tensorflow_datasets/ted_hrlr_translate/pt_to_en/1.0.0. Subsequent calls will reuse this data.</span>
tfds.core.DatasetInfo(
    name=&#39;ted_hrlr_translate&#39;,
    version=1.0.0,
    description=&#39;Data sets derived from TED talk transcripts for comparing similar language pairs
where one is high resource and the other is low resource.
&#39;,
    homepage=&#39;https://github.com/neulab/word-embeddings-for-nmt&#39;,
    features=Translation({
        &#39;en&#39;: Text(shape=(), dtype=tf.string),
        &#39;pt&#39;: Text(shape=(), dtype=tf.string),
    }),
    total_num_examples=54781,
    splits={
        &#39;test&#39;: 1803,
        &#39;train&#39;: 51785,
        &#39;validation&#39;: 1193,
    },
    supervised_keys=(&#39;pt&#39;, &#39;en&#39;),
    citation=&#34;&#34;&#34;@inproceedings{Ye2018WordEmbeddings,
      author  = {Ye, Qi and Devendra, Sachan and Matthieu, Felix and Sarguna, Padmanabhan and Graham, Neubig},
      title   = {When and Why are pre-trained word embeddings useful for Neural Machine Translation},
      booktitle = {HLT-NAACL},
      year    = {2018},
      }&#34;&#34;&#34;,
    redistribution_info=,
)

</pre>
</div>
</div>

</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="prompt input_prompt">In&nbsp;[7]:</div>
<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="c1"># 输出test</span>
<span class="k">for</span> <span class="n">pt</span><span class="p">,</span> <span class="n">en</span> <span class="ow">in</span> <span class="n">train_examples</span><span class="o">.</span><span class="n">take</span><span class="p">(</span><span class="mi">5</span><span class="p">):</span>
    <span class="nb">print</span><span class="p">(</span><span class="n">pt</span><span class="o">.</span><span class="n">numpy</span><span class="p">())</span>
    <span class="nb">print</span><span class="p">(</span><span class="n">en</span><span class="o">.</span><span class="n">numpy</span><span class="p">())</span>
    <span class="nb">print</span><span class="p">()</span>
</pre></div>

    </div>
</div>
</div>

<div class="output_wrapper">
<div class="output">


<div class="output_area">

    <div class="prompt"></div>


<div class="output_subarea output_stream output_stdout output_text">
<pre>b&#39;e quando melhoramos a procura , tiramos a \xc3\xbanica vantagem da impress\xc3\xa3o , que \xc3\xa9 a serendipidade .&#39;
b&#39;and when you improve searchability , you actually take away the one advantage of print , which is serendipity .&#39;

b&#39;mas e se estes fatores fossem ativos ?&#39;
b&#39;but what if it were active ?&#39;

b&#39;mas eles n\xc3\xa3o tinham a curiosidade de me testar .&#39;
b&#34;but they did n&#39;t test for curiosity .&#34;

b&#39;e esta rebeldia consciente \xc3\xa9 a raz\xc3\xa3o pela qual eu , como agn\xc3\xb3stica , posso ainda ter f\xc3\xa9 .&#39;
b&#39;and this conscious defiance is why i , as an agnostic , can still have faith .&#39;

b&#34;`` `` &#39;&#39; podem usar tudo sobre a mesa no meu corpo . &#39;&#39;&#34;
b&#39;you can use everything on the table on me .&#39;

</pre>
</div>
</div>

</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="prompt input_prompt">In&nbsp;[&nbsp;]:</div>
<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="c1"># 从语料中构建subwords-level</span>
<span class="n">en_tokenizer</span> <span class="o">=</span> <span class="n">tfds</span><span class="o">.</span><span class="n">features</span><span class="o">.</span><span class="n">text</span><span class="o">.</span><span class="n">SubwordTextEncoder</span><span class="o">.</span><span class="n">build_from_corpus</span><span class="p">(</span>
    <span class="p">(</span><span class="n">en</span><span class="o">.</span><span class="n">numpy</span><span class="p">()</span> <span class="k">for</span> <span class="n">pt</span><span class="p">,</span> <span class="n">en</span> <span class="ow">in</span> <span class="n">train_examples</span><span class="p">),</span> <span class="n">target_vocab_size</span> <span class="o">=</span> <span class="mi">2</span> <span class="o">**</span> <span class="mi">13</span><span class="p">)</span> <span class="c1"># 2192</span>
<span class="n">pt_tokenizer</span> <span class="o">=</span> <span class="n">tfds</span><span class="o">.</span><span class="n">features</span><span class="o">.</span><span class="n">text</span><span class="o">.</span><span class="n">SubwordTextEncoder</span><span class="o">.</span><span class="n">build_from_corpus</span><span class="p">(</span>
    <span class="p">(</span><span class="n">pt</span><span class="o">.</span><span class="n">numpy</span><span class="p">()</span> <span class="k">for</span> <span class="n">pt</span><span class="p">,</span> <span class="n">en</span> <span class="ow">in</span> <span class="n">train_examples</span><span class="p">),</span> <span class="n">target_vocab_size</span> <span class="o">=</span> <span class="mi">2</span> <span class="o">**</span> <span class="mi">13</span><span class="p">)</span>
</pre></div>

    </div>
</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="prompt input_prompt">In&nbsp;[9]:</div>
<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="c1"># test </span>
<span class="n">sample_string</span> <span class="o">=</span> <span class="s1">&#39;Transformer is awesome.&#39;</span>

<span class="n">tokenized_string</span> <span class="o">=</span> <span class="n">en_tokenizer</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="n">sample_string</span><span class="p">)</span>
<span class="nb">print</span><span class="p">(</span><span class="s1">&#39;Tokenized string is </span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">tokenized_string</span><span class="p">))</span>

<span class="n">origin_string</span> <span class="o">=</span> <span class="n">en_tokenizer</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="n">tokenized_string</span><span class="p">)</span>
<span class="nb">print</span><span class="p">(</span><span class="s1">&#39;The original string is </span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">origin_string</span><span class="p">))</span>

<span class="k">assert</span> <span class="n">origin_string</span> <span class="o">==</span> <span class="n">sample_string</span>

<span class="k">for</span> <span class="n">token</span> <span class="ow">in</span> <span class="n">tokenized_string</span><span class="p">:</span>
    <span class="nb">print</span><span class="p">(</span><span class="s1">&#39;</span><span class="si">{}</span><span class="s1"> --&gt; &quot;</span><span class="si">{}</span><span class="s1">&quot;&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">token</span><span class="p">,</span> <span class="n">en_tokenizer</span><span class="o">.</span><span class="n">decode</span><span class="p">([</span><span class="n">token</span><span class="p">])))</span>
</pre></div>

    </div>
</div>
</div>

<div class="output_wrapper">
<div class="output">


<div class="output_area">

    <div class="prompt"></div>


<div class="output_subarea output_stream output_stdout output_text">
<pre>Tokenized string is [7915, 1248, 7946, 7194, 13, 2799, 7877]
The original string is Transformer is awesome.
7915 --&gt; &#34;T&#34;
1248 --&gt; &#34;ran&#34;
7946 --&gt; &#34;s&#34;
7194 --&gt; &#34;former &#34;
13 --&gt; &#34;is &#34;
2799 --&gt; &#34;awesome&#34;
7877 --&gt; &#34;.&#34;
</pre>
</div>
</div>

</div>
</div>

</div>
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
</div><div class="inner_cell">
<div class="text_cell_render border-box-sizing rendered_html">
<h3 id="&#26500;&#24314;Dataset">&#26500;&#24314;Dataset<a class="anchor-link" href="#&#26500;&#24314;Dataset">&#182;</a></h3>
</div>
</div>
</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="prompt input_prompt">In&nbsp;[&nbsp;]:</div>
<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="n">buffer_size</span> <span class="o">=</span> <span class="mi">20000</span>
<span class="n">batch_size</span> <span class="o">=</span> <span class="mi">64</span>
<span class="n">max_length</span> <span class="o">=</span> <span class="mi">40</span>

<span class="c1"># 把句子转为subword形式</span>
<span class="k">def</span> <span class="nf">encode_to_subword</span><span class="p">(</span><span class="n">pt_sentence</span><span class="p">,</span> <span class="n">en_sentence</span><span class="p">):</span>
    <span class="n">pt_sentence</span> <span class="o">=</span> <span class="p">[</span><span class="n">pt_tokenizer</span><span class="o">.</span><span class="n">vocab_size</span><span class="p">]</span> <span class="o">+</span> <span class="n">pt_tokenizer</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span>
        <span class="n">pt_sentence</span><span class="o">.</span><span class="n">numpy</span><span class="p">())</span> <span class="o">+</span> <span class="p">[</span><span class="n">pt_tokenizer</span><span class="o">.</span><span class="n">vocab_size</span> <span class="o">+</span> <span class="mi">1</span><span class="p">]</span>
    <span class="n">en_sentence</span> <span class="o">=</span> <span class="p">[</span><span class="n">en_tokenizer</span><span class="o">.</span><span class="n">vocab_size</span><span class="p">]</span> <span class="o">+</span> <span class="n">en_tokenizer</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span>
        <span class="n">en_sentence</span><span class="o">.</span><span class="n">numpy</span><span class="p">())</span> <span class="o">+</span> <span class="p">[</span><span class="n">en_tokenizer</span><span class="o">.</span><span class="n">vocab_size</span> <span class="o">+</span> <span class="mi">1</span><span class="p">]</span>
    <span class="k">return</span> <span class="n">pt_sentence</span><span class="p">,</span> <span class="n">en_sentence</span>

<span class="c1"># 过滤大于max_length的句子</span>
<span class="k">def</span> <span class="nf">filter_by_max_length</span><span class="p">(</span><span class="n">pt</span><span class="p">,</span> <span class="n">en</span><span class="p">):</span>
    <span class="k">return</span> <span class="n">tf</span><span class="o">.</span><span class="n">logical_and</span><span class="p">(</span><span class="n">tf</span><span class="o">.</span><span class="n">size</span><span class="p">(</span><span class="n">pt</span><span class="p">)</span> <span class="o">&lt;=</span> <span class="n">max_length</span><span class="p">,</span> <span class="n">tf</span><span class="o">.</span><span class="n">size</span><span class="p">(</span><span class="n">en</span><span class="p">)</span> <span class="o">&lt;=</span> <span class="n">max_length</span><span class="p">)</span>

<span class="c1"># 使用tf.py_function封装py函数</span>
<span class="k">def</span> <span class="nf">tf_encode_to_subword</span><span class="p">(</span><span class="n">pt_sentence</span><span class="p">,</span> <span class="n">en_sentence</span><span class="p">):</span>
    <span class="k">return</span> <span class="n">tf</span><span class="o">.</span><span class="n">py_function</span><span class="p">(</span><span class="n">encode_to_subword</span><span class="p">,</span> <span class="p">[</span><span class="n">pt_sentence</span><span class="p">,</span> <span class="n">en_sentence</span><span class="p">],</span> <span class="p">[</span><span class="n">tf</span><span class="o">.</span><span class="n">int64</span><span class="p">,</span> <span class="n">tf</span><span class="o">.</span><span class="n">int64</span><span class="p">])</span>
</pre></div>

    </div>
</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="prompt input_prompt">In&nbsp;[&nbsp;]:</div>
<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="n">train_dataset</span> <span class="o">=</span> <span class="n">train_examples</span><span class="o">.</span><span class="n">map</span><span class="p">(</span><span class="n">tf_encode_to_subword</span><span class="p">)</span>
<span class="n">train_dataset</span> <span class="o">=</span> <span class="n">train_dataset</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">filter_by_max_length</span><span class="p">)</span>
<span class="n">train_dataset</span> <span class="o">=</span> <span class="n">train_dataset</span><span class="o">.</span><span class="n">shuffle</span><span class="p">(</span>
    <span class="n">buffer_size</span><span class="p">)</span><span class="o">.</span><span class="n">padded_batch</span><span class="p">(</span><span class="n">batch_size</span><span class="p">,</span> <span class="n">padded_shapes</span><span class="o">=</span><span class="p">([</span><span class="o">-</span><span class="mi">1</span><span class="p">],</span> <span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]))</span>

<span class="n">valid_dataset</span> <span class="o">=</span> <span class="n">val_examples</span><span class="o">.</span><span class="n">map</span><span class="p">(</span><span class="n">tf_encode_to_subword</span><span class="p">)</span>
<span class="n">valid_dataset</span> <span class="o">=</span> <span class="n">valid_dataset</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span>
    <span class="n">filter_by_max_length</span><span class="p">)</span><span class="o">.</span><span class="n">padded_batch</span><span class="p">(</span><span class="n">batch_size</span><span class="p">,</span> <span class="n">padded_shapes</span><span class="o">=</span><span class="p">([</span><span class="o">-</span><span class="mi">1</span><span class="p">],</span> <span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]))</span>
</pre></div>

    </div>
</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="prompt input_prompt">In&nbsp;[16]:</div>
<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="c1"># test</span>
<span class="k">for</span> <span class="n">pt_batch</span><span class="p">,</span> <span class="n">en_batch</span> <span class="ow">in</span> <span class="n">valid_dataset</span><span class="o">.</span><span class="n">take</span><span class="p">(</span><span class="mi">5</span><span class="p">):</span>
    <span class="nb">print</span><span class="p">(</span><span class="n">pt_batch</span><span class="o">.</span><span class="n">shape</span><span class="p">,</span> <span class="n">en_batch</span><span class="o">.</span><span class="n">shape</span><span class="p">)</span>
</pre></div>

    </div>
</div>
</div>

<div class="output_wrapper">
<div class="output">


<div class="output_area">

    <div class="prompt"></div>


<div class="output_subarea output_stream output_stdout output_text">
<pre>(64, 38) (64, 40)
(64, 39) (64, 35)
(64, 39) (64, 39)
(64, 39) (64, 39)
(64, 39) (64, 36)
</pre>
</div>
</div>

</div>
</div>

</div>
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
</div><div class="inner_cell">
<div class="text_cell_render border-box-sizing rendered_html">
<h2 id="&#24037;&#20855;&#20989;&#25968;">&#24037;&#20855;&#20989;&#25968;<a class="anchor-link" href="#&#24037;&#20855;&#20989;&#25968;">&#182;</a></h2><h3 id="&#20301;&#32622;&#32534;&#30721;">&#20301;&#32622;&#32534;&#30721;<a class="anchor-link" href="#&#20301;&#32622;&#32534;&#30721;">&#182;</a></h3><p>根据公式写函数</p>

</div>
</div>
</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="prompt input_prompt">In&nbsp;[&nbsp;]:</div>
<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="c1"># 求角度</span>
<span class="c1"># pos.shape: [setence_length, 1]</span>
<span class="c1"># i.shape: [1, d_model]</span>
<span class="c1"># result.shape: [setence_length, d_model]</span>
<span class="k">def</span> <span class="nf">get_angles</span><span class="p">(</span><span class="n">pos</span><span class="p">,</span> <span class="n">i</span><span class="p">,</span> <span class="n">d_model</span><span class="p">):</span>
    <span class="n">angle_rates</span> <span class="o">=</span> <span class="mi">1</span> <span class="o">/</span> <span class="n">np</span><span class="o">.</span><span class="n">power</span><span class="p">(</span><span class="mi">10000</span><span class="p">,</span> <span class="p">(</span><span class="mi">2</span> <span class="o">*</span> <span class="p">(</span><span class="n">i</span> <span class="o">//</span> <span class="mi">2</span><span class="p">))</span> <span class="o">/</span> <span class="n">np</span><span class="o">.</span><span class="n">float32</span><span class="p">(</span><span class="n">d_model</span><span class="p">))</span>
    <span class="k">return</span> <span class="n">pos</span> <span class="o">*</span> <span class="n">angle_rates</span>

<span class="c1"># 求编码</span>
<span class="k">def</span> <span class="nf">get_position_embedding</span><span class="p">(</span><span class="n">setence_length</span><span class="p">,</span> <span class="n">d_model</span><span class="p">):</span>
    <span class="n">angle_rads</span> <span class="o">=</span> <span class="n">get_angles</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">arange</span><span class="p">(</span><span class="n">setence_length</span><span class="p">)[:,</span> <span class="n">np</span><span class="o">.</span><span class="n">newaxis</span><span class="p">],</span>
                            <span class="n">np</span><span class="o">.</span><span class="n">arange</span><span class="p">(</span><span class="n">d_model</span><span class="p">)[</span><span class="n">np</span><span class="o">.</span><span class="n">newaxis</span><span class="p">,</span> <span class="p">:],</span> <span class="n">d_model</span><span class="p">)</span>

    <span class="c1"># angle_rads.shape: [setence_length, d_model / 2]</span>
    <span class="n">angle_rads</span><span class="p">[:,</span> <span class="mi">0</span><span class="p">::</span><span class="mi">2</span><span class="p">]</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">sin</span><span class="p">(</span><span class="n">angle_rads</span><span class="p">[:,</span> <span class="mi">0</span><span class="p">::</span><span class="mi">2</span><span class="p">])</span>
    <span class="n">angle_rads</span><span class="p">[:,</span> <span class="mi">1</span><span class="p">::</span><span class="mi">2</span><span class="p">]</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">cos</span><span class="p">(</span><span class="n">angle_rads</span><span class="p">[:,</span> <span class="mi">1</span><span class="p">::</span><span class="mi">2</span><span class="p">])</span>
    
    <span class="c1"># position_embedding.shape: [setence_length, d_model]</span>
    <span class="c1"># position_embedding = np.concatenate([sines, cosines], axis = -1)</span>
    <span class="c1"># position_embedding.shape: [1, setence_length, d_model]</span>
    <span class="n">position_embedding</span> <span class="o">=</span> <span class="n">angle_rads</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">newaxis</span><span class="p">,</span> <span class="o">...</span><span class="p">]</span>
    <span class="k">return</span> <span class="n">tf</span><span class="o">.</span><span class="n">cast</span><span class="p">(</span><span class="n">position_embedding</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="n">tf</span><span class="o">.</span><span class="n">float32</span><span class="p">)</span>
</pre></div>

    </div>
</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="prompt input_prompt">In&nbsp;[65]:</div>
<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="c1"># test</span>
<span class="n">position_embedding</span> <span class="o">=</span> <span class="n">get_position_embedding</span><span class="p">(</span><span class="mi">50</span><span class="p">,</span> <span class="mi">512</span><span class="p">)</span>
<span class="nb">print</span><span class="p">(</span><span class="n">position_embedding</span><span class="o">.</span><span class="n">shape</span><span class="p">)</span>
</pre></div>

    </div>
</div>
</div>

<div class="output_wrapper">
<div class="output">


<div class="output_area">

    <div class="prompt"></div>


<div class="output_subarea output_stream output_stdout output_text">
<pre>(1, 50, 512)
</pre>
</div>
</div>

</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="prompt input_prompt">In&nbsp;[66]:</div>
<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="c1"># 画出位置编码的图</span>
<span class="k">def</span> <span class="nf">plot_position_embedding</span><span class="p">(</span><span class="n">position_embedding</span><span class="p">):</span>
    <span class="n">plt</span><span class="o">.</span><span class="n">pcolormesh</span><span class="p">(</span><span class="n">position_embedding</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">cmap</span><span class="o">=</span><span class="s1">&#39;RdBu&#39;</span><span class="p">)</span>
    <span class="n">plt</span><span class="o">.</span><span class="n">xlabel</span><span class="p">(</span><span class="s1">&#39;Depth&#39;</span><span class="p">)</span>
    <span class="n">plt</span><span class="o">.</span><span class="n">xlim</span><span class="p">((</span><span class="mi">0</span><span class="p">,</span> <span class="mi">512</span><span class="p">))</span>
    <span class="n">plt</span><span class="o">.</span><span class="n">ylabel</span><span class="p">(</span><span class="s1">&#39;Position&#39;</span><span class="p">)</span>
    <span class="n">plt</span><span class="o">.</span><span class="n">colorbar</span><span class="p">()</span>
    <span class="n">plt</span><span class="o">.</span><span class="n">show</span><span class="p">()</span>
    
<span class="n">plot_position_embedding</span><span class="p">(</span><span class="n">position_embedding</span><span class="p">)</span>
</pre></div>

    </div>
</div>
</div>

<div class="output_wrapper">
<div class="output">


<div class="output_area">

    <div class="prompt"></div>




<div class="output_png output_subarea ">
<img src="
AAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0
dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nOydd5xU1fmHn/femdneKywsvVooIojY
sHeN3cRYYjSJ5afGaDSJJjHFmKIxicagMdEUe0zAYLCgoqCAhY60pe7Csn13dqfdO+f3x70zO7ss
MMAusHiez+c4t98z63Dmzvc97/cVpRQajUaj+WJgHOgOaDQajWb/oQd9jUaj+QKhB32NRqP5AqEH
fY1Go/kCoQd9jUaj+QKhB32NRqP5AtGjg76IbBCRpSKySEQ+drfli8ibIrLGfc3ryT5oNBrNgUJE
nhaR7SKybCf7RUR+JyJrRWSJiIxP2HeNO06uEZFruqtP++NJf6pSaqxSaoK7fg/wtlJqGPC2u67R
aDSHIn8FztzF/rOAYW67EfgjOA/HwA+BScBE4Ifd9YB8IOSdC4Bn3OVngAsPQB80Go2mx1FKzQHq
d3HIBcCzyuEjIFdE+gBnAG8qpeqVUg3Am+z6yyNpPN1xkV2ggDdERAF/UkpNA0qUUlvd/duAkq5O
FJEbcb75yEhPO6pNpTN21AAWrdzI2JHlbP5sOQMOG8SiTU1k5OXQr2UrDY0hSscdxpI1W/GmpXNY
oYmyLVa1eGhrqKOobwllqomq9TWkGkLhyIGsbxUattdhen0UFuVSU12HikbJKshnSEEaoc0V1NcF
sBXkpnvJLC+hzZvNhuoWSvLTKUgBq2YbrdtbaLGiAKSbBhm5KaQUFWCn5VD52XJ8ImSkmKTmpuHN
yyOamkVL2KahNUxbwMIKBbEjYYjajB9SRLS1mXBLG5HWMOFwlFBUYStFFBDAFPCIUFCWixUIYQUt
7JBNOBrFihI/NpZvbQDZh40kbCvCVpSwZRO2okRt5bRoFBW13eYsjyn1IR4vYnrBMFGG6bwiRBXY
CpRy+rWqYisiAiII7qthtK8bBiIGIoI3xUQpQCmUew1nHZTzH2KZ4kpFycxKRUQQwBDBvQ2CYAjO
PndbVWVd/F0r5wKdPpHt60MG9UFinzf3P+KudVx3WLl2SzKfeQAOH9a/y+0iO25bumpT0tcFOHJk
edfX7mLb4s+Tv/bYnVy3KxbtwXWdaw/Yg2tvTP66ozped9HKjahAXa1Sqijpi3TCyO6nsIJJHasC
dcuBxIOnueNcspQBmxPWt7jbdrZ9n+npQf84pVSliBQDb4rI54k7lVLK/ULYAfcPNw3gqCMPU0vN
Scyd+zg5k29izgeP8Z2MUTz28pMU3DKLYy85mwdn/4RXZ6zhu3Pn0vf8Byk97CjmX5+B3VTHie8V
8MlL/+CK++/gwfBr/PirTzI808e1Lz/JVxek8upjfyWzdCDX3ngOf3zkn0SCrRx/9WW8/JXDWX/b
V/nH35fSFIly0ahSjv39d1hcdjJXP/I+d145hqsHm9Q+8TPm/2EO79S0ATA+J5VJ5w1jyI3X0HL4
WXw/ezR9UzxMHpjDiAuPpO8ll9A68mTe29jE8x9vZsmSaravW41/2wasoJ8FL91I2/w3qHxvEVUL
K9m4qZkNbRHqwzbhqMIUyPGaFPpMrrn7AmqXrKNuVS0NFY1U+sPUhGwaIjYBO4rt/nV9hnDWy2+w
qSnIhtpWNta1UlXXRmtziLamEMG2MKGWRsJtTVgBP1awlbnfKccsKMXMK4aMXKIpWUTTcomYKbRF
orRGogQsRXPI4uQrfoTp9WF4fBgeL4bHh5mShunxxZcNjw+Pz0u/YQVY4ShWxMaK2NhWFCsSJWpF
se0othUlakexLYuoFWbySSPweQx8HtN5NQ1SPIa7rWO774d/RUVt5zPkfnk5y85r1H0FePwv38MQ
MEUwRDAN50ul87oIGAhHnX93h2vtihlvPAK0D/Kxn9TibjASRugBU2/d7fUSefu9P3Q5wBtdbCw+
/pakr/veB491WO/qHjHyp9yc9HUBPpj7eNLH5h57U9LHzu103ZzJNxFZ9JfkvzW6wgriGXF+UodG
Fv0lmCBd9wp6VN5RSlW6r9uBV3G0qWr35wvu6/ae7INGo9HsESKIYSbVuoFKIPFnYT9328627zM9
NuiLSIaIZMWWgdOBZcB0IBaJvgb4T0/1QaPRaPYccX+x7r51A9OBq91ZPMcATa78PQs4XUTy3ADu
6e62faYn5Z0S4FX356wH+KdS6n8ishB4UUSuBzYCl/VgHzQajWbPcJ/0u+dS8hxwElAoIltwZuR4
AZRSTwAzgbOBtUAbcJ27r15EfgIsdC/1gFJqVwHhpOmxQV8pVQGM6WJ7HXDKnlxrxfYwk++6mndH
TmLyrY/y0dEncNkRxVw2z/mmnX5eHrfdtJIf/OwczvrjfIJNtTx7x/G8c9bpRF55jSUzf0H55HN5
6IzBvD3iOQK24rRvTGZ+6mjee/0VVNRm9AlH8/LMVbTVVTHg2PO4+9ThRN98isXTV1MTshmfm8qo
yyZgjT2Hv/9vDePG9eH0oQVEF/yTDW8uZ2lTiHBU0T/Ny+CheZSdMBaGTWJFTYBMj8GgDC/FRxRT
OOEwVPkRbGoO88nmRtZvaaa5toFgQzVW0A9AuGI5jas307i+gcatfmpCNn4rSjjqCPQ+Q8gwDfJ9
Jq2VtbRt99NWG6ApaOG3orTazrExPd8UpzUEIjS0halrDVPnDxMKWIQDFuGQRSTYhh0OYIcCRK0w
KmpjpGdhpGYgvjSinlSULx3lSSFsKScgHFWE7SghK4qYsZ+8BmKYGF4fhvsT2PD4EMPE9HgQEeyY
dm9HUVEnkKyiiqhyXpVSRKMqrp2bhmAahvMq4q530RKipCoa3fXn03avnaSe337d3ev5e0JXgd29
oSs9X/bh4t3UrV6JAGJ2z6CvlLpyN/sV0GWARCn1NPB0t3QkgZ4O5Go0Gk3vQgSjm570D0b0oK/R
aDSd6C5552BED/oajUaTSDdq+gcjetDXaDSaBATB8HgPdDd6jF7hshlqaeTtk4O8saWZ2eeYvLqy
hmPmz+G1x57ipz+5nrdO+DJH56VRe+3Pmf/8i0y87FJGz32MV1fWcPtjH6Jsm/uuP5rah25nZmUz
Z/XPps+3f8x3X1pC7eqFFB82hfvOP4zKT98ho6g/Z586lGPSG1k+7TUWNgTJ8RqMn1xG0UVf4a31
jby3cDNXTOhPWet6Kl+fzeplNVSHLNJMYXS2j37HDiRj0slsM3KZu7GekhQP/QfmUjphKKlHTKbB
V8CirS18urGB+q0ttG7fRLi1CQDTl0Zwwzoa11bRvKWZmpBNs+UkWoETxM30GOR43UDutjr81a20
1QdoikTjAV87IfPUFMFnCPXBCNubQ9T5QwQDEcKBCOGQ5SRIhQJYYSeIG7Ui2JEwRkY2RkYWUV8a
UW8aypNCROEEcKMKy4a2iE3QisaDtrEgriQGcc1YMFcwPQYqihOwjSpsO+oGbVU8OUu5QVwVtVG2
HQ/U+kwnASuWmNU5kGuIdAi07ioxC7oOfu6MPYmJxu6XTGLW3vBFDrLuF/bvPP39jn7S12g0mk70
1gE9GfSgr9FoNImIdNuUzYMRPehrNBpNAsKh/aTfKzT9/uV9+Pmxt3D/7y/n4QnXc9ddJ3L099+k
z7hTub763/y7ooEvT/8xF/10NukFfXntW5N4/ua/MzwzhfUfTOfIc87nqvwaZjz6Pvk+kxMevJRn
1iuWv/0+vowcTjvzcE5Kr8UOBxg8aTK3HT+Qxuf+wEdzt+C3ohyTn8aor06lKv9wnp63gaqVnzN1
YA6BOa+y/q11rPaHsRUMTPfRf3wpfaYegzXgKD6pauGdldsZmumlz/g+5IwdS6TPYaxtCPLxxgaq
NjfRsn0rYX8DUSuMGCa+jBwaVm+maWMz9XWBeGJWonFaLDErPT+Nlq1+2uoC1IdtmiI2QVdvT0zM
8hlCmmlQ7w9T3xqmMZaYFbKJhCysgB87HCAacfX8WHJWRhbKl4HypoM3lagnhWAsMctWBK0oQStK
W8TuYLQmhomRkJRleHwYhmCaBqZpxBOzbCuKihLX8uPafkzTtx1dP2a0ZhqCJ0HD76Dri2C6Ynd3
J2ZJ/LrJJ2btTM/v6ph9RSdmdTNiYHp8SbXeiH7S12g0mkTk0H7S14O+RqPRJCDoefoajUbzheJQ
HvR7haaf3VBJaaqHx4d/DYAl1zzEmndeZfYvzubRrzzO1SeU86g1no3zZvDtOy+l6vavsLAhyJX3
n0l2v+H89YaJfHbzXSxuCnLByQNpPfsOfvPcYvzVGxh4zFR+cOpQtv7x1xQMHc+3zhtFeeWHLH7q
A1a2hOif5uXwL43Cd+rV/HtVDcs/20rzltWkrXmfdf/5kCWbmqgP2+T7TEaWZtD/pNH4xk1lbbPi
3TW1VFY00PeIYkonjcYz+hgqQyafbm1m8YZ66qv9BBq2xefoe9IySckppHFtNc1bmtkWjM3Rbzda
y/Q4en5OqoeMknRaq1tpaQrRFIkSjCoCdrsxW+wcnyGkGhKfox8KWIQCESIhi0gwiB125ujb7jz9
mJYuaVkoXxrKm0LUm0bIisb1/LCtaIvYtEWihOxo3GgtZrwW1/PdOfuGaWB4DMQQopZTMEUp5RRM
6WS0FjN8i7XdGq25c/QNQ+J6/u7m6MfYmZ7fmWTn1u9O949d52DV8w80B0XX9Tx9jUaj+SKh5R2N
RqP5wiAiGN7eOTMnGfSgr9FoNIlowzWNRqP5YqEH/QPMtmo/1237mOzzf41/wTQKb/8jU2+4ntZb
L6fVjjL+9dc558JfMuSkC7mnTxXf++siLh5ZQPBrP+OKQRWUz3mCH7+1nqPzUhn38I+4dsZKNsyb
RU75KG65+HDKVv6X6U9+xNgfX89Vhxey7vY7+KCiAVOEY0fkM+Cqy1gUyOK59z6jZtUnRK0w22e8
SsWcTWwORPAZwvBMH+VT+pF3/Ek05g7hgxU1fLyqhobKKvpMGEjG2MkE8gezbEMT89bUUlvZQmvN
JkItDU4ilMeHLz2b9IIyGj9porolTEOkPYhrCmR6DLLdQG5GSQaZJRnUr2mgPuwkcCVW14KOQdw0
06C+NURLa5hQMEI4YBEJWe1Ga25iVjQhgKp8jsma8qZjYRC2o25zgrghO0rIsp3KWQkGa2anIK7p
MTA9hhMo9RjxRCzbUnHjtVhiVqLRWodAbqfErA4JWm5iVqxy1q6CuLHELOg6YBsjMTFrb4O43W20
tj/oBV3cLxi94X/WXtIrZu9oNBrN/kJEECO5luT1zhSRVSKyVkTu6WL/IyKyyG2rRaQxYZ+dsG96
d7y/XvGkr9FoNPsT0+ye52ERMYHHgNOALcBCEZmulFoRO0YpdUfC8bcC4xIuEVBKje2WzrjoJ32N
RqNJROjOJ/2JwFqlVIVSKgw8D1ywi+OvBJ7rhnexU3rFoF9SkMa4Xy2n7KhTOGWWYKak8fqp8Njz
K/jOY1dy4q/nYQVb+fe9J/G/M/4PnyGc/NIvufyJ+Tx8cjEzb3mGcFRx9l2n8JaM4M1X5wIw5rTJ
XDcygyUPPsmc2jZ+cs5o7P88woJ/rWRb0GJ8bipHXHccbWPO5cmPNrL+s1W01VWRllfK2hmLWdwU
ImAr+qZ6GDaqkP6nToCRU/hsWytvLN9G9aZG/NUbKJo8juigcaxrCLFgYwPrNjTSVF1LsKEaK+gH
wJeRQ1peKVn5mTRu9bMtaHXQ6NNMI260lpOfSmZxOhmluTQFLZoiUVrt6A5Ga4lma5keoS5mtBaw
CIcsIsE27HAAOxSIJ0RFI+2JUVFvOsqXTtSbSiiWlBVVhO0oIddoLfYa0/ANo2NylunxYJpOUpaT
nOUUUInarpbvJmjFErMS9XxwdHJTZKeFU2JJVYaboLUrEvV82HliVkzPT2RPk4Z29Q8r8Vr78g/w
UDNaOygSs4i5bHbboF8GbE5Y3+Ju2/G+IgOAQcDshM2pIvKxiHwkIhfu5VvqgJZ3NBqNpgO7f4BI
oFBEPk5Yn6aUmraXN74CeFkplfh0MkApVSkig4HZIrJUKbVuL68P6EFfo9FoOuLKO0lSq5SasIv9
lUD/hPV+7rauuAK4OXGDUqrSfa0QkXdx9P59GvR7hbyj0Wg0+5NulHcWAsNEZJCI+HAG9h1m4YjI
SCAP+DBhW56IpLjLhcAUYEXnc/eUXjHoB0oGsPa911j68NnMe/YZpv/+Bp6adD0Xjyzg9Qnf4rNX
n+PKW64i47E7mbGlma/dcizT/ENY9J9/sfa2G3hreytfOqoPObf/hnue/YT6isWUTzyNRy8+ksYn
f8Jb720CYFx4NZ/8diYLG4KUpnqYcOZgci/+Ov/6vJb3P9xEw4ZlGB4f+UPHs3xVPduCFjlegyPy
0xhwykjSJ5/NxkgGb6+uYd3aOho3rybYVIPvyBPYRjbztzTx4Zpa6rY5c/TjRmupjtFaRmEpuUXp
VAYsmq3oDsXQ830mhSkeMoozyOybRWZZEfVhm1Y7uoPRmimOlp9qGGR6nNbWGiYciLhma2GsgH+H
YugxPR9wzdbS2oumdDBaa2+BiN1urObxOc3rvrp6vmka8UIq8Xn6dkfjtUSTtcSWqOX7Omn7RsIc
fVOSN1pTUXu3Rmv7Mke//Ro7n6Pf3Xp+b+Zg0fPB6YvpkaTa7lBKWcAtwCxgJfCiUmq5iDwgIucn
HHoF8LxSSiVsGwV8LCKLgXeAXyTO+tlbtLyj0Wg0nehOp1Kl1ExgZqdt93da/1EX580Djui2jrjo
QV+j0WgSEHc22KGKHvQ1Go2mE3sQyO116EFfo9FoOnEoD/q9IpC7YeM2vvfzO3h35CQmX3U1BT+/
gQ1tEU78aBa3/OBvlE8+lycmWPzpodlcMCCHtPue4CePvI4vI4fnXlzBmJxUjn3ifm6f8TmrZr9O
dr/h3HTFkQzfNJuPfvM2G9oiTClIo+KRX/Puku0AnDAsn2E3fJkV0pen317HtuWfYIcDZPcbzpAj
S1ntD2EKDM/0MWjqAIpPPYXm4tG8t6Ge95dVU7N+C221VaioTaB4BEuqW/lgTQ3btzTTvHUDwaZa
olYYw+MjJSuP9IIycoszGNgni1rXQM1WToJVmilkewyKUkwyStLJ6ptJZlkRGWVFNEW6CuI656S6
AeBMj5CZ4iHYFiHkGq3Fgrh2KIAdDmJ3qlYFoLzpRMRDyIoSdKtmBSNR2iLtiVlBO0ogbLuJWDsa
rcWCt6bHwDCdBC3bUk4AdydGa0CHfnRltBavpiXEE7NiP8m7CqomJmbtqrpVotFa5207Y0+CuD0Z
sNxfFbP2YA5770Sc95hM643oJ32NRqNJQHAeTg5V9KCv0Wg0icihba2sB32NRqPpRG8uLr87esVv
GG96FjetnMYbW5qZfRY88uSn3PPEV5jy208JNdXy+o9OY+Zx12GKcPrMR7nw9x9Su3ohJ1xxHn4r
ykXfO40308Yx/YX3UFGbo846nm8dlsniH/+et7a30j/Ny6TrjuaD55ZS5RqtjbnxRAITvsSjcyqo
+PRzWms2k5ZXSv/DR/PVyQMI2Ir+aV5GHVHMgLMmwREn8/HWVl5bspWt6+vxV2/ACvoxPD7WNoSY
W1HH6ooGGiq3dWm0ll2YQ1FJJkf2z93BaC3bY8aN1rL6ZJJRmktmWRGeojI3Mauj0VqseErMaC3H
a5KSnUI4YDmJWQlGa3Y4uIPRWoyY0VowwWgtlpAVM1oLhJ0W1/M7Ga0ZHiNutBYrpLIzo7XEPsRN
3zolZ8WTtEwjruN7DcPR9jv9Q40lZu1Mz9+d0Zoh3avBH6xGaweag63rjuFacq030uPdFhFTRD4T
kdfc9UEiMt8tKPCCm5qs0Wg0BwexyQFJtN7I/viuug0n/TjGQ8AjSqmhQANw/X7og0aj0SSJYJhG
Uq030qO9FpF+wDnAU+66ACcDL7uHPAN0i0e0RqPRdAein/T3id8CdwNRd70AaHRNiGDXBQVudIsH
fFySEuSHt7/CDx+/kocn3shXjinj2ZHXsfjfz3PbvdcjD1zPa1tb+Mb9Z/CLrX1Z9J+XGXzCBbx4
9TiumDoQz7ce4q6nFlJfsZjBU87ksUuPpObR+5j5zkZMgVOn9KP/Td9mYUOA/mlejvnSCLIvvYnn
lm3ng7kbadiwDNOXRtHICZx+TDlnDc0n32cytjSTQWceQeqx57E2mMrMFdWsW11H46bPCTbVIIZJ
Wl4JH25u5KM1tdRWNbvF0OsBx2gtNa+ErOI+5JdkcHhZDiOKMncwWitKMSlK95JRnEFWvxyyyktI
KS3FU1q+wxz9mJafYTomazleE1+6l5RsH6FAhHAggBXwEwn6XaO18A5GazGc+fmOyVrIUrSEdjRa
8wct2sL2Lo3WTI/76mr8yRqtxYq0J2O0ZhgdDdd2ZrSWyO6M1mKbO8/bT2Rfjda6Q4vfn3p+d89N
P9j0/BjdWSP3YKPHBn0RORfYrpT6ZG/OV0pNU0pNUEpNKCwo6ObeaTQaTdeI0HUyYBetN9KTUzan
AOeLyNlAKpANPArkiojHfdrfVUEBjUajOSD01gE9GXrsSV8pda9Sqp9SaiCOV/RspdRXcHyhL3EP
uwb4T0/1QaPRaPYUIbmn/N76xXAgkrO+CzwvIj8FPgP+fAD6oNFoNF0iAj5tw7BvKKXeBd51lyuA
iXtyfu2yVVw6fhyPDbkWHy8z7H9vcPZ5P+SIcy/jvrRPueeJhXzlmDLqr32Qh6//PZmlA3n6juOo
/M7VTHjqt5z3j0Wsfe81CoaO54fXHkW/hX/npd/PoSpocV6/bMbeex3z7H74DOGk8aUMvfmbzPVn
8fSsT9m69EPscIDC4UczZkIZV43vR2H1Ig7PTmHI6UMoOv1sanKH8uayauYt3Ubt+vW01TlGaylZ
+WSWDOKtFdVs29hI89YKgk21TnDSl0ZqTiEZReXklWQyon8uR5TlMDQ/nZmu0VqmxyDPa1KUYpLV
N5Psfk61rIy+xXhKyiGneIcgrs9oN1rL8Rqk+UxS81JJy0slFIi0V8uKhB2jtYgTzO0qkOtUyooS
snasltWakJgViNgdgrimxzFYixmtiUg8Scs0jR2M1qJWGGV3DOJCu+lah6QsjxEP3noTErQSDbAS
g7h7Y7SW+AC3N0Zr8XO7MFrr7iBusvfvnut9QYK44pj8HapoGwaNRqNJQDi0NX096Gs0Gk0i0nv1
+mQ4dIUrjUaj2QucJ30jqZbU9UTOFJFVrvXMPV3sv1ZEakRkkdu+nrDvGhFZ47ZruuP99YonfVvB
gP+9wVnn3IN/wTSG3PkaGUX9mffdyTzRdyKjslKY9PqrHHHfbNrqqrj7gVsZ+8lf+OWfPyX78kzm
vfwSvowcLvvyiVycvZ337vkLc+sCjMlJ5ZjvnkHN2Iu4/9lPubM4k3F3XMDm/lN46OWlrP/4U4JN
NWT1GcLg8SP5+rEDGW7UUTv9RUYcU0b/807BGn0yc9Y0MP2TSrZWbMdfvQE7HMCTmklmyUAKy0uo
WFdPU1UlbXVV2OEAYpj4MnLIKContyiDsr5ZHNk/h5GFGZRmOP9LEvX8nOIMsvtlkV1eTFZ5CWZJ
OUZxOXZWyQ5Ga2luUlamxyDba5Lm6vmpealYAT92OOC+dl04JZGQpRzDNSuaoOdHCVlO4ZRYYlas
iIrh8bUXTYmZrZkS1/cNQzA9gm1FidpRbMuKF07pKjErRgfDNRG8RruOHzNaM2XHn+S70vOdWEFH
o7WdFU7prPPvKQeicMrBrucf7HTXk76ImMBjwGk4yagLRWS6UmpFp0NfUErd0uncfOCHwARAAZ+4
5zbsS5/0k75Go9EkYEh7BvjuWhJMBNYqpSqUUmHgeeCCJLtyBvCmUqreHejfBM7cqzeVgB70NRqN
phPODLHdN6AwZhfjths7XaoM2JywvjPrmYtFZImIvCwi/ffw3D2iV8g7Go1Gs7+QLqTCXVCrlJqw
j7ecATynlAqJyDdwjChP3sdr7pRe8aRfethgJn/jacqOOoVTZgnVS+cw45GrmXvsaVQFI1z72gOc
8dfPWf/BdCZdcRk/HObnxRv/TFPE5jePv02goZpx553FQ2cMZtnd9zJzZS2lqR5Ou+pIMq/9AT+b
vY6V73/GUbeeAGffwm/f38CS91fSvGU1qTlF9DtyHF89aTBTyzMJz/4Ha/79KcMumowx8TwWbm3j
34sq2bSqlqZNKwi11GN4fKQX9iW3XzmDh+RTV1mLv3oDkdYmwCmckl7Ql5ySQorLshk/II/RRZn0
y/aRGap3C6E7en5BXiqZfTPJ6pdHVnkJvrIBePsOJJpZSMiXBSTq+UKG6czPz/EapOT4SHX1/NS8
DKygn0ig3Witq8IpMcQwCdpOQXR/2MLvzsdvi9j4Q1a7nh+xCYQtDK8P0+NxLGdjc/I90mG+vmE6
lrWJhVN2ZbQW0/vjhmsJ8/I7G63F5ul3VThlZyRTOGVPjdYSr9P5/P1ltNYbJp4c7CGCbszIrQT6
J6zvYD2jlKpTSoXc1aeAo5I9d2/oFYO+RqPR7C9iyVnJtCRYCAxzi0f5cCxppne8n/RJWD2f9voj
s4DTRSRPRPKA091t+4SWdzQajSYBQbrNhkEpZYnILTiDtQk8rZRaLiIPAB8rpaYD/yci5wMWUA9c
655bLyI/wfniAHhAKVW/r33Sg75Go9EksIea/m5RSs0EZnbadn/C8r3AvTs592ng6W7rDHrQ12g0
mg4c6jYMvULTX1ETIdRSz9KHz2bes89w9wO3kvvgDby4dDvf/uk5/KJtDB/+458MPuEC/vfNibx7
4U18VB/gylMHUfP5RwybegF/ueYoah+6nf/MWIOtFGefWM6g797Hk0vrmTlzOfUViym8/i6eXrSV
mW+tpXb1QkxfGsWjJ3HeiYP40shCZN6LrH5hDkuW1ZBx8sWstbJ5eXEVS5dtp379CgIN1fFqWbn9
h9N3UB5TRxXTUrW2Q7WstIK+ZJf2o6BPJuMH5HFEn2wG5aaSb4Tw1G8kx03KKkr3kt0vi5zyXLIH
9iG1f3+8fQdiZ5diZxbREDgBOkAAACAASURBVHSCiV1Vy0rPTiE1103Myk0jJTeLSNBJzooZre0q
iCuG2WW1rNbwjkHceOUsM9FobSdJWh6jQ7WsxGByV0HcRMO1xGpZsQQtb2y7G9Dtiq4Ss3Z4z7uo
lmUIHWzXdhfETbxmjJ0Fcfd2bNHVsnoQXURFo9FovjjE/PQPVfSgr9FoNJ3Qg75Go9F8QTAO8SIq
veKdBZsbeeOp23l35CQmX3U1d9e/zG//9DHfvGgES790P7958BnyB4/hP9+fypqvXcyLS7dz4eA8
xj/9R0rHTOW335hEyZuPMuPR96kKWpw9LJ9xP7md1/3F/PGlZVQvnYM3I4fXa1N5asZKqhbPQUVt
CocfzXHHDeSao/qRv2EuG16cwbIPNrPaH6IyawjTV1Yzd1EV1WtW0VqzOV44JbvfCEoH5HHyYSVM
7pdHoKE6XjglLa+ErJIBFJZlc8TAfMb0y2FEYTp90g08dRsIVyyn0GdSmupx9Px+2WQP6kNGeRne
PgNReX2JZhXTEIrSGLTjhVPa9XyDjDRPB6O1tIIcUguysUMBolZkl4VTYnq+GGZcx28JtxdO8Qct
x2wtZOEPRuKGazG93uM12w3WEgqnxLV+Q3ZaOKWznh/D5zHwGsZOC6eYRsciKrszWou/110UTknU
83d2frLowintHPR6PmhNX6PRaL5ICHFfnUMSPehrNBpNJw5lK2k96Gs0Gk0CAjud/nso0CsG/X79
SzFuvow3tjQz+yz43tinOX9oPvlPvsKZN/wZMQweu+9CMh67kydeWskx+Wmc+vKD/GhxlO9/6wRO
2P4O/73jORY3BTm1OIPjHrqG5X1P4MdPLWDDgtmIYVJ+9FQemr6CDQvmEWltIn/wGEZPHsatxw9m
cOsaKp9/jlUzVrOsOUTAVry+po4Z8zezdfVG/Ns2ELXCeDNyyC4bTunAIo4dXcxxA/MZUZBC1Apj
eHyk5hSSWTqI/D5ZDCvPZfyAXA4rzqQs04u3bi3W+mW0rl3j6PllWeQOzCF7UCnZA/vg6TsIKeyH
lVVCk2XQELTZ2hKKm6zF9PycVE/cZC29MJ1UV89PLchx5ujvonBKop4vhok/bOMPW+1Ga0Fnjn5L
yIrPzw+EbayI7Wj5icZqMQ0/Yc6+x/Ugj+n50d0UcYkXRk8wVzNE8JrSoXBK4nKyej7sqOd3Zb4G
ziBgiOyRnt/Vg2JnPb+75+gf7Hp+r8H9rB2q9IpBX6PRaPYXAniTLIXYG9GDvkaj0SSg5R2NRqP5
IuFOCT5U0YO+RqPRJBCL4Ryq9ArhKrdpK0++toYfPn4lD0+8kVFZKZz06TtM/d4smqvW8f37ruW0
xU/yp4dm0zfVy+V/+RbP2qP50xOvcUNhNe99/SFmVbcyPjeVU39yAdunfI3bnl/E6vfexQr4KR0z
lavOHcnncz6kra6KrD5DGHbMkXz7lGGM8dRQ+9JfWPHiIhY2BGiKRClKMXn+w41s/rySxs0rsYJ+
PKmZZPcZQsngMsaPLmbqsEIOL04nbfsqxDCdIG7JIArL8hk8IJdJg/MZU5JN/ywvqY2bsDetJLD2
cxrXbCa/JIPcAdnkDCwhZ0gZ3n5DMUsHYef0oVVSaQjZVLWEqGwJkpYQxM3zOUlZ6YXppBekkVqQ
FQ/ienLzsd1qWbEAalfEgrim10dLyKmY1eKarMWN1sJ2/DUctrGt6I7GarGELI9TLcuTUEy6c1LW
zozWYq3dXM2IV8lKTNRqr5zV/j6SNVlLXI4FcTsEd/fto7vTf2DdH3Tt7ut1/6DXm8ZRx9hv9603
op/0NRqNJgFxHygOVfSgr9FoNAkc6vKOHvQ1Go2mE71VukmGXvEbZuu2Fr575/E8NuRaAK5a/DIT
fzqXLQv/x1V3fI3/s+fxhxv/hinCDQ9fwrvDL+f+R96kadNKPrzmTv69qo7hmT7Ou/sU7Ct/wC2v
LGXpm3MINGyjePQUzj9rBDdP6kfL1nVkFPVn6DETuf3MEUwtsmj5959Z/vf5LKhqoSZkk+M1GJ+b
yvplW2ncsIxIaxOmL43M0oEUDxnMEaOKOX1kMeP6ZJLTtJ7wsrmk5hSRWTKIgv7F9B+Qy7HDChnX
J5uBuT4yWqtRm1cSXL2MhtWbaVhTQ97gXHIGFZMztAxfv8F4+g7GzimlzZNJXcBmW0uYrS0htjQE
yPYY5PtM8n2mY65WmEZ6YRpphVmkFuSQXpyHNy8PI7tgl3p+YlKW6fXFk7M6JGUFLfwhi5ZgJK7n
WxEbKxLF8Bh4vB1N1zxeI15YJabnp3iMdsO13ej5MRL1fI9pJGj47Xq+12z3S0lGz49fW3av5xsi
e6VHd3fhlJ3epxcMUL3pwVloN/DbXUvqeiJnisgqEVkrIvd0sf/bIrJCRJaIyNsiMiBhny0ii9w2
vfO5e4N+0tdoNJpEurFGroiYwGPAacAWYKGITFdKrUg47DNgglKqTUS+BfwSuNzdF1BKje2Wzrj0
iid9jUaj2V84mn5yLQkmAmuVUhVKqTDwPHBB4gFKqXeUUm3u6kdAv258OzugB32NRqNJIGbDkEwD
CkXk44R2Y6fLlQGbE9a3uNt2xvXA6wnrqe51PxKRC7vj/fUKeac4L5UPvvwgP/3Wg/gXTOO4Z7ex
ctbLnPGtG3h8xHaePPHnNERsbv/xWaw58y6+9cAbbF8xlyEnXcgLv7uNvqleLrppMjm3/4ZvvLKM
D2e8h796A4XDj+b0c8Zw78mDSZn9FGl5pQyeNJmbzhnJuQNSCb7yW5b+dQ4frm2gKmiR6XH0/GGn
DKShYjHBphoMj8/V84czcmQhZx5WwtF9syhsq8JaNpfajz4lo2gCeWWl9C3PZcqwQsb3yWFwbgrZ
wVpkywqCa5dQ//lG6ldto2F9I0POHEHe8P6kDhiCt3w4Vk5fAil51LZZbPOHqWwOsqmhjY11bRzr
debop+c7Wn56YTppBZmkFeU5en5uLkZuMWZe0W4LoRseH2LGlr34w5ZbLKVdzw+EnSIqgaAV1/Ot
iO2YqiXMzzdMiev5aT4zruf7PGZS8/MhZrgW7VLP9yYsO0XRZY9M0VTU7lAIHXau5+8Nyer5+5wH
0ANa+aE8cyUpBPZgxmatUmpCt9xW5CpgAnBiwuYBSqlKERkMzBaRpUqpdftynx570heRVBFZICKL
RWS5iPzY3T5IROa7QY0XRMTXU33QaDSaPSU2ZbObArmVQP+E9X7uto73FDkV+D5wvlIqFNuulKp0
XyuAd4Fxe/3GXHpS3gkBJyulxgBjgTNF5BjgIeARpdRQoAHn54xGo9EcJIhr5737lgQLgWHuw64P
uALoMAtHRMYBf8IZ8LcnbM8TkRR3uRCYAiQGgPeKHhv0lYPfXfW6TQEnAy+7258BukWn0mg0mu6g
O5/0lVIWcAswC1gJvKiUWi4iD4jI+e5hvwIygZc6Tc0cBXwsIouBd4BfdJr1s1f0qKbvTlf6BBiK
M21pHdDo/iFgF0ENNyByI0Cf9NSe7KZGo9HEcWwYui+uoZSaCczstO3+hOVTd3LePOCIbuuIS4/O
3lFK2e4c0344U5dG7sG505RSE5RSEzIGDeeb//cbyo46hVNmCZ+89A+mXHMt/znF5O8n38Zqf4ib
7zyR+msf5MpfvEPVJ7MYcOx5TLt1Cvk+k8u/No4+9/2OO/+7ilmvzKF5y2ryB4/hpHMm8MPTh5H7
4T/49JcvMeiY47jx3FFcMTIX67+Ps/TP7zBvWQ2bAxEyPQZjclIYedIABl80lUDDtoQg7khGjC7i
gjF9ObZ/DiXhaqylc6j9cCFV8yvI79+fvgOdIO5RZTkMzU8lN9KAbFlBaPVn1C9bT8OqrTRUNFJT
GyBveDmpA50grp3Tl1B6AXUBi+2tYTY3BdjUGGBjXRtb6tvI95lk5qWSXphGRkkGGcVZpBXlkVaQ
g68gHzOvGDOnACMrn6gV3uHv3DmIa3p8GB4vhseHP2TR1BbpEMRtCVqEEpKyrIhN1IrGK2d5fCaG
KfEErcSkLJ/HbK+clWQQF4hXzdpZENcbq57Vxad5ZxW5oD2IG6ugFf+buK+xJ7l9iWseyCDu3lz/
Cx/EdRFJrvVG9svsHaVUo4i8A0wGckXE4z7tdxnU0Gg0mgOJsc9fyQcvPTl7p0hEct3lNJyMtJU4
2tQl7mHXAP/pqT5oNBrNniLoJ/29pQ/wjKvrGzgBjNdEZAXwvIj8FCf9+M892AeNRqPZY3qDn9He
0mODvlJqCV3MKXXnm07ck2tVbNhG+ZensPThs8mZfBOTr7qaty7M5h8TrmRxU5D/u/042m5/lIt+
OptNH75G+eRz+dMdxzFhxfOUXjuW/g9O485ZG3n1uXdp3LCM3IGHc+J5k3nwnFEUf/wCnz74d97+
ZCvf+PVorjmiEHvG71j02CzmLapmQ1uENFMYk5PCESeWM+zSqXiPvwTD8wsySwdSNHQ0w0YXceHY
MqaU59InUkN02Rxq535E1fx1VC+rofSCXE4YUcTkAXmMKkynwGrAqFxBePVn1C1ZR93KSurWNLB9
eyvbghZpQ4bhGzgSO68/oYyieFLWpqYgmxoDVNS0srG2lebGIJl5qWQUZziavqvnpxfnkVJc6Oj5
ecUYOYVE03J2+LvuSs83vL4Oer4/GInr+eGQhRWJErWcZkWipKZ7u9Tz03xmBz3fZxp7pOerqO0a
rslu9fzOevSu9PwYiXq+ITvX8/fmJ7HW83spvfgpPhmSHvRF5FhgYOI5Sqlne6BPGo1Gc8AQkp6D
3ytJatAXkb8BQ4BFQOxRSQF60NdoNIccWt5x/CBGK6VUT3ZGo9FoDgYO4TE/6UF/GVAKbO3Bvmg0
Gs0BR5dLdCgEVojIAhxPHQCUUufv/JTuw5OWycpHz+GdkZOYfOujvPOlTJ49ygni3nbnCbTd8Xsu
eOBtNs6bQfnkc/nznScwcfk/mf71J7hw3Txud4O49RWLyR88hhPPm8yvzh/tBHF/9gxvLaiiKmhx
95FFRGf8js9+P5P3P9sWD+KOz03lyJMHMuzyU/CecBmfWznxIO7oI0q4cGwZJwzIpa9VQ3Tpu9S8
P4/KeWupXlbDqpYwU0cV7xDEDa1Y0GUQtzZsx4O4wYwiatwg7oaGwA5B3LbmEBnFGR2SsnYWxO0c
yN1VENdMScP0+HYbxI0laNlWNOkgrs9j7FEQF0g6iJuow+ogrmZfOITH/KQH/R/1ZCc0Go3mYOJQ
LjSS1KCvlHpPREqAo91NCxLd4DQajeZQQbqxXOLBSFJfaCJyGbAAuBS4DJgvIpfs+iyNRqPpneiM
XMfc/+jY072IFAFv0W6R3KMc3i+L1wdNYE5tG7PPgqeOuorV/jDfue90ar7+EF+6/w0qF85k8AkX
8Lc7T+CwD5/g5ZufZW5dgNdnVPDfF96madNKCoaO54wvHcvPzhpB/ty/svBn/+TtRdVsC1oMTPcS
eflXfPbYG7y/tN1kbXxuKkecOpChl5+GecLlrAxm8OLiKkqGHcbhR5bwpXFlHNc/h9LQVqzF71Dz
wXy2zFvLthW1rPVHqA5ZnDcwnxGFaRSE65xKWSsWULtkHXUrqqhfW09NbYDKgEVDxMZvRbHyBxBK
L6CmzaKy2TFZW1/vVMpK1PPbWkId9PyMPgXtJmsFpUh2IdHULEfTT8mK/z2T0fNjhmtd6fkxk7WY
nm/b0aT1/BSPsUd6vlPhKjk9P6bFJ6Pnw64rZXXW82Uv/4XvTs/v7gfKXjoOHVQIWt4BMDrJOXUc
2n8XjUbzBWZvv+R7A8kO+v8TkVnAc+765XTyh9ZoNJpDAtHJWSil7hKRi3HKdQFMU0q92nPd0mg0
mgOD4NRwOFRJ2ntHKfUK8EoP9mWn1C9dxUdGH374+JU8PPFGmi2bex+5mM/OuJvr7vk31cvmMOqM
S3jhjuMo/c8v+Pt3/8WnjUGmFqXzrWdfw1+9geLRU7jwoqN54PShpP7vD3z081eYvbKWmpDNkAwf
J0wuY+GvZ/LBmnqqghY5XoOj89I47OwhDLr8XIzJF7G4yeS5zzbz/qIqxo/vw4Vu0ZSi1k1EPptN
9fsLqPpoPVWf17HWH6Y6ZBGwFaOL0skNVMOmpQRWfhrX8+vWNrC9PsC2oB3X88NRRVtqPrWtFpXN
ITY1BVlf1xrX8/2NQVqbQwT8IUItzWT2yUnQ8wswcosx84ocPT8tB5WWg52SSVvE0coT9XzT63OX
vZi+NAyvD9Pjc5Y9PhrbwgTCNoGg1aFoihW2idoK252rb8eKqLhafmLRlDSfic+MrTttT/R8oIOe
7zXb9fvOer5pJK/nQ9d6fndp+YnXj6H1/N7DoSzv7FKXF5EP3NcWEWlOaC0i0rx/uqjRaDT7Dycj
N7mW1PVEzhSRVSKyVkTu6WJ/ioi84O6fLyIDE/bd625fJSJndMf72+WTvlLqOPc1a1fHaTQazaFE
dz3nu/VEHsMpIrUFWCgi0zsVOL8eaFBKDRWRK4CHgMtFZDRwBXAY0Bd4S0SGK6W6/umaJMnO0/9b
Mts0Go2m9+PIhcm0JJgIrFVKVSilwsDzwAWdjrkAeMZdfhk4RRx96QLgeaVUSCm1HljLHtYi6Ypk
p10elrgiIh7gqH29uUaj0Rx0JJmY5Y75hSLycUK7sdPVyoDNCetb3G1dHuPWDm8CCpI8d4/Zpbwj
IvcC3wPSEjR8AcLAtH29ebKEo4ofv3YPv/GehI+X+d6Lt/Fc3wu55+6/0bxlNUdd+hVevfkY7N9+
myd/9Q7rWsOc1y+bkx/7Olf/eCllR5/NdZcewd3HlRP820+Y89DrvLWpCb8V5fDsFKZMHcCob17C
zy58iJqQTVGKyaT8dEZeNIr+l1yAmnghH1S18fynG1mwqIrqNRU8cPmlTCzLIrduNaFP3mLrnI+p
/GgTWyoaWesPUxu2CUcVpkCefzPR9YtpW76IuuUV1K6opqGikW2NQbYF25OybNe4urrNCeJuaAyw
sa6Niho/VfWBeBA32BYm1NJMpK2J9NEFZJQW4C2ImawVQWZB3GTN9qbTGo7SFom2J2QZJqbXScBK
rJTlcQO4sSQtf9AiHLa7DOJaYRvbdpKzonYUnxvA9XkM0n1mh6SsxCCuz2N0COK2B23bg7iJgddo
1E46iNvVk9fOgrgxkg3i7mnQVQdxey+iFLKbz00CtUqpCT3Zn+5ml0/6SqkHXT3/V0qpbLdlKaUK
lFL37qc+ajQazX5FVDSplgSVQP+E9X7uti6PcVWUHJwE2GTO3WN2N3tnpLv4koiM79z29eYajUZz
8KFARZNru2chMExEBomIDycwO73TMdOBa9zlS4DZbsGq6cAV7uyeQcAwHA+0fWJ38/S/DdwI/KaL
fQo4eV87oNFoNAcd3VQkUCllicgtwCzABJ5WSi0XkQeAj5VS04E/A38TkbVAPc4XA+5xLwIrAAu4
eV9n7sDup2ze6L5O3dcb7Qt9DhvEV6rGMPNPj+JfMI271hTy57ufIGqFOfMb1/LCFaOouPVK/vnc
cvxWlC9P7Mvk393NwpLjGXriPO7+yli+XK7Y/svb+fDxD5hT2wbAlII0jr5wBENuuJbGUadTE/o5
/dO8TByQzciLx9Ln4ktpHX4isysaef7jzSxbUs32dZ/j37aBEwfkkLr5E1o/epPKOYuoWlDF+i3N
bA5Y1Cfo+TleE/vz+bQsW0zdsvXUraqloaKRSn+YmpCTlBWw2/V8nyFU1AfY1BRkQ20rFTV+qhsC
tDaHaGsK0eYPEWltItzWhBXwk1lWhLewBMMtmkJGLtHUHKKp2UTMFNrCNq2RKG0RtVM9P9FkzUxx
dH2Pz0soZGGFY8VSbDcZyymgkqjn25aVoOUbu9TzfYmGawl6fueErGiCpuo1DQxht3p+Z0k/GT1/
dwZr+6q9d3W61vMPcpRK9ik+ycupmXSyrVFK3Z+wHMRxMO7q3J8BP+u2zpD8lM1LRSTLXf6BiPxL
RMZ1Z0c0Go3mYKEbNf2DjmSnbN6nlGoRkeOAU3F+jjzRc93SaDSaA4WCqJVc64UkO+jHfiefg2O2
9l/A1zNd0mg0mgOIojsDuQcdyRquVYrIn3BSiR8SkRT2o5/+yjqbJb//E+WTz+WUWcJH//wDmaUD
ueP2i7hnsJ95p53NSwuqyPeZfO3SUYz+5UP8syaPX/xuHo/fPJnjqWD1vT/nnZc/Z3FTkByvwfGF
GYz52kTKrvsGFdmj+OvcjYzKSmHCkcWMuGwieed9ha05w/nf8u08v2AzG1Zsp75iGW11VUStML4V
b1M/dzaVH6xg6yfbWFvbRlXQoiliYytHm8/xGvRN9VI/fz51yzZQv7aBuo1NVAacAuhNEUf7T9Tz
Mz0Gq+taqdjeysa6VuoaArQ1h5z5+a1Bwi31RIJ+rIAfOxzEWzwEs6AUM684XixFpeUQxENbOEpr
JErAitISsuKGaolz8x39Pq2jnu+ap0VCdnw+vqPpq3gBlZimr6I2USsc1/PTfJ4OBVNiOr5pyA6a
Puxez1e23UHP7zxXH9r1fCNB3d6dnh87D5LT8/fGf6un5+Z3dQ9Nd6Ag2jsH9GRIduC+DCf6fIZS
qhHIB+7qsV5pNBrNAeRQ1vST9dNvE5F1wBmu09v7Sqk3erZrGo1Gc4DopQN6MiQ7e+c24B9Asdv+
LiK39mTHNBqN5oCgFETt5FovJFlN/3pgklKqFUBEHgI+BH7fUx3TaDSaA0VvlW6SIdlBX2ifwYO7
vN9iSIGmBqbc8VXeuHUyOZNvonzyuUz79vFM/vxFXj3mcd7a3sqYnFQu+M5U8r7zCHfNWstLL79F
9bI5HHN2LfN//lfe/LCSqqBF31QPJx1ZzJgbTyb9/BuZ25LBtFmrWDB/Cy+cNpDhV5yC98TL+NzO
55VPKnl94RYqV1fRtGklgYZtAKRk5VP92nQqP1zDtsXbWdXiVMnyW84HJc0UCn0eytI89ClIY9v8
NdStaWD79ta4wVpTxKmSBU5ptlgQN9tjsryymY21rTQ3BmlrDtHWEiLU6ifS2tQhiGuFAnhKyjFy
CuMGa9GULNosRVvEptWKEohEaQpaNIWsHYK48QCuWzXL40vBMA08PhOP18RKMFuz3eBth8QsK+wE
ciNhJ4DrJmR1DuLGm2lgiOyySlYsiKvshOQsw9hlQlYsgCuSXAA38X67C+LubQGlZIK4+1KdSQdw
e5LuTc462Eh20P8LMF9EYnVxL8SZq6/RaDSHHl/0QV8p9bCIvAsc5266Tin1WY/1SqPRaA4U3WzD
cLCxOz/9VOCbwFBgKfC4a/Kv0Wg0hyTCF1vTfwaIAO8DZwGjgNt7ulOd6duvlHdOj/DmyEkce9vv
+PcNR9Pwo2/w8OMfURWM8KVh+Zz4h5upOPJSvvzH+SyZ9R7+6g3klI/irese4Z1tfgJ2lPG5qUw+
czDDb7iC8DGX8s+VtTz97lLWfbqOho3LGP2rG7HHncPsjc28+Nk6Pl60leo1a2iuWocV9GN4fKTm
FJLdbwRrZjzO5g2NrG+NdCiYkukxKElx9Pzisizyh+VTtaCKquYQ24I2zVbHgimmQJppkOkxyPOa
5PsMZlY1428KEmgJE/CHCLU0xg3W7HAQKxwgGgkTtcJIfh/sNNdgzZNGWzhKwFK0RqK0hm2aQhZN
wQj+sI3pS91Rz08wWDNNA4/XxPAYeLwG4ZC1U4O1mJYfS85K85k7NVgzDcFnGngNwTBklwVToKOe
r6L2bvX8uC6fpNCdqOfvqmBKouS+L5mIh5qevw9d7yUosHvnzJxk2N2gP1opdQSAiPyZPfByFpH+
wLNACU5i8zSl1KMikg+8AAwENgCXKaUa9rzrGo1G0wPEbBgOUXb3ABOJLeyFrGMBdyqlRgPHADe7
1d3vAd5WSg0D3nbXNRqN5qDhi5yRO6ZTbdxYrVwBlFIqe2cnKqW2Alvd5RYRWYlT1PcC4CT3sGeA
d4Hv7u0b0Gg0mu7lCxzIVUqZ3XETERkIjAPmAyXuFwLANhz5p6tzbsSp2kVZTiYPTb6Z2rDF22co
3jv2JF5Ztp2SFA+3fm0sQ376G57e6OHhn81m04I3ASiffC6XnjOSGec+Rr7P5NTyPI687hhKrvoG
a9IGM+3Ndbw5dyNVyxbhr96AitpsHX4GMxdt48WPNrHp8xrqK5bQVleFitp4UjPJKO5PfvkwSgfm
suzVejYHInF93mcI+T6TkhQP5Vk+8gbnUjCikLzh/Xl/VkXcYC1gt1fkaZ+bb5Dj6vk5WSk01rQS
8IfjBmvhtibsUAAr2IrtavkxPdzOKkKlZBFQJoEEg7XGgIU/7MzP94csmkNWgn7ftcGax2vi8Rlx
bb+1ObTD3PyYhh/T8+OavtfcQc+Pmax5DQNTnGIoXkN2a7AWX3a3ew1jh+LniXq+IcnpzJ3n8Cdj
sHYwafl7fv/uvdehr+UncAgP+j3ulCkimcArwO1KqebEfW4dyC7rkimlpimlJiilJhRkpPV0NzUa
jcbhELdh6NFBX0S8OAP+P5RS/3I3V4tIH3d/H2B7T/ZBo9Fo9gyFsiJJtX1BRPJF5E0RWeO+5nVx
zFgR+VBElovIEhG5PGHfX0VkvYgsctvYZO7bY4O+OL9j/wysVEo9nLArsfL7NcB/eqoPGo1Gs8co
9teTfjKTWtqAq5VShwFnAr8VkdyE/Xcppca6bVEyN03WhmFvmAJ8FVgqIrHOfA/4BfCiiFwPbMTx
6tdoNJqDAoXqEFvqQXY7qUUptTphuUpEtgNFQOPe3rTHBn2l1AfsPI/klD25VlVVE0W5+dz820t5
eOKNrGsNc16/bE7+/XVUTvk6Z72wmEWzPqB5y2qy+gxh9NRj+cH5h3FKdhNPZKcwZeoARn3zEtRJ
V/PSqjr+9O9FrFu0kbq1nxJpbcKTmklOv+H84p11fPRZFdtWr6N56zoirU2IYZJe0JesPkMpHljK
0CH5nDSymJX+cDwhpAf3dAAAH8FJREFUK8drxA3WSvtkkj8sj/zhfckbNYCUQSPZHJix04SsbI9B
vs+kMMVDemEaGSUZtDQECLU0E2lrItzatENCVmJA0krLpzUSpS1eIcumJWzRFLTwh22aQhH8QYum
tgje1EwnOcsN4naVkBUL6Joew62WtfOErHggN2rHK2ftLCErFsz1mEZSCVmJ7C4hq3PVrK7oyoht
TxKy9jQAeyCDuN0dwIUvWhCXPamcVSgiHyesT1NKTUvy3KQmtcQQkYk4ZWrXJWz+mYjcj/tLQSkV
2t1Ne/JJX6PRaHohak+km1ql1ISd7RSRt4DSLnZ9v8MdlVL/396Zh8dxl3n+81Z1t9SSbN2SZcu2
HN8mISGHQ8jAhCSQwJJjsyEkMAyzS8bDcj/AkIQsEOaZeTYwswnLwgLmZiYDA4E8BAiYJORYjhCc
xE7s2I4dH/FtWZd1tNRdXb/9o37dqpa7pZYPSe1+P89TT1f9qrqqfnbr7erve4lI3qAWe5424F+B
9xiTDS26g+DLIgasJfiV8A8T3bAafUVRlDDGnLSTdvRU5spC+0TksIi0GWMOjhfUIiKzgV8Cdxpj
ngqdO/MrYUREvgN8oph7mrLm5oqiKKWByUqXEy0nyYRBLSISAx4Avm+MuX/MvkwUpBCUu99UzEVL
4km/ua6S/7rll3xhc5oY9/PJj76O9s/cwz0b+vnGp3/D/mcexonEOOsN1/Hua1fygYvbqXzye2z4
8v284663Un/TGl6UuXz1F9t48g+vcPDF5xjs3AtATWsHjYvP4axXtfCrX2+ld/cLJHoOY/w00epa
ZrV2UNfeQduiei5d3sylixp4VUs1G31D3BXqoy5zKiPMr62gYWkDDUsaqV+5kJolS4h2rMRvXEhf
alQfjLtC3B3V8htiLrNqK6hpqaaqKU5N22yGug7nFFgbm5AVpmc4ndXzM81SBqymP5gMtPzeoRQD
Ix5uLJ6TkBWJuYGmH0rICmv7WU0/1CwlnJDlhz788ZCm71oNP+oGRdJGdX3J6s0TJWSFt6NuSMPP
k5AV1vjHMtEf5qnW8vMx3jmKLRJXLJqQdQrIRO+cfvIGtYjIhcD7jDG32rE3AI0i8jf2fX9jI3Xu
E5FmAt/pBoKKyBNSEkZfURRl6jCTceSe+FWM6SJPUIsxZj1wq13/N+DfCrz/8hO5rhp9RVGUMIap
CtmcFtToK4qi5DCp6J2SoySMvte+iPPv2cqOJx5i4Om1POKu4u33PMtLTzxCcrCP5hWv5dI3ncPn
3rKCpV3Psuv2/8EzP9rEU90JPnHfg9z7/EHuf+Jp9mx8kb59L5FOJqisbaau42zmr5jHla+Zy9tW
tvKG7/076WQCNxanqnEus9uXM6ejnvOWNfH6xY2cP3c2HbOjRA9vGy2uVhWhcWEtTcsbqVvWTt3y
RUQ7ViBti/Hq2ulJB//EMUeIu0K1O6rl11dHiTdVUdNSRXVrNfGWeqrnNJD41aFs4/NCWr44LuK4
9A6PxuVn9Pz+kUDLHxgO1geGU/QPe0Sra0fj8LMN0B2r44/R9yMOXjIVXD+dG5dv/DTpzLZ9Ispo
+hkNP2qboEfdQMePOoJrNf1iYvPD22OLq40dg+O18WKcbGP1/PG0/BPV3gvp+arlz2BOYfTOTKQk
jL6iKMrUoU/6iqIo5cPURe9MC2r0FUVRQhhMto/zmYgafUVRlDD6pD/9vLz7ENHHfk77RW/minXC
8+u+xsDh3dQuWMmFN1zLXdes4tKKwxz+2t/z62/+kd8fGaQ7mWZOZYR3fvvP7Nywm+5dG0kN9hGt
rqW+42zmrjiLS8+byzVnz+HCtmpmH3kR46ezDtyWhS0sW9zAXy5v4eL2WhbXVxDv2Y2/fiPHNm3g
3NoKWubNChKybHG1WMcK3HnLSNe3c8yponPQY9+xIWoiQXG1etsdqz4Wobq1iqqmKqpbqqhqqaW6
rZF4cz3R5laSP9mRt7haBnFcnEgMcVwODozQbztj9SdHHbi9iRQDwymGkmkGhj2SyTSxikhOAlY2
OSvq4kYEx3WIhZKs0iOJvMXVsg7ddCg5K+rmLa6W6ZjliGTXi3XgZnBDna3CxdVyHLuMOjOLzZQs
JiGr3By4UOZOXAgcuankdN/FaaMkjL6iKMrUMTXJWdOFGn1FUZSxqLyjKIpSJhhzKoqpzVhKwuhH
Kqu57R8/yu1/2UHtJe9nVttiVt/8bu68bhVX1g3Q/f3P8eg3f8/vXumjcyRNc4XL29pmseKGlfzz
Az/LNkppXHI+c5Yt5qJz27j2nDYuaZ9FXfd2RtY9zK4n19O84mqaFrSybGkjl61oYfW8OhbXx6jp
34//3HMMbn2eo8/v4OiLh1n++vk5jVLc9kDL73Nr6Ex47D82yO7eBHu6hphbGTmuUUpGyw8SshqJ
Njbh1rfg1jfjJTZMqOW70aAZysH+kaDAWiJF31CQhDUw4tE/nMpq+V4qjZfyicWjxzVKiUSd47T8
iohDPBYhnUxMqOVn7rMi4oyr5WcStdwCunuhPzLjpyfU8mG0wcpk/1iL1fJPVuZWLb+00OgdRVGU
csEYTFqNvqIoSllgjMFPedN9G6cNNfqKoihhDPqkP92cPX82H97+LR7/u9/wuo98ic9cs4o3xI9y
5Dt38cg3/8D/OzhAdzLQ8q9pn83KG8+m/cbr8c+/BnP5bTQtu4i2ZYu4xMblXzS3htqjWxn59SPs
emI9B/68jz0v9/D6ez6ejctfVFdBdd8r+M89x8CWQMvv2tZJ9/ZuDhwb4aYv3kzF4lXZuPxep4rO
IY99xwZ5pS/Bzs5B9nQNsu/oEB+fVXGclp+Ny29swm2cg1vfAtX1+PHavMXVxmr5TiSKE4nxSs9Q
UFht2KMvkcyJy0+NBHp+Ou3jJdNUVkfHjcsPmpvbbdc5rlFKPi0/o31Wuk7BuHxHglj7TIPz8PzG
0/IzZPwA42n5MPk2cJnjp1PLP5Hznw49X8lFjb6iKEqZYIzB13r6iqIo5cOZHL2jjdEVRVHC2Oid
YpaTQUQaRORhEdluX+sLHJcWkQ12eTA0vkhE/iQiO0TkP2wT9QlRo68oihIiE71TzHKS3A48aoxZ
Cjxqt/ORMMacZ5drQ+OfB+41xiwBeoD3FnPRkpB3up/fymc+3EPMER69yrDrix/gJ7YzViJt6KiK
cuXyRlbcdAGtN7yD3vmr+enOHn5430Zec9312c5Y5zRXEt31JwZ+9AjbntjIgWcOsfNAP3sTKbqT
aT5z1fJsZ6zU75+hZ/MmujbvomtrFz07e9k7lKJzxOOY5xO//O2k69rpTEfoHPLY09vP3r4Eu6wD
91DXEIPHRhg8NsKc81pyOmPFW+qJ1Dfj1LcQaZyDX1WHXzELP15L0gm+rDOdscRxcaIxHOvMzThw
3Yo4biTGvp5EtjPWwLAXJGIlfZuQZR25nsH3fKpnV+Z0xorHXCqsEzfswM2MeclEtjhaPgfu6HpQ
cC3TGWu0S1auAzdw7o5fFC1vUlqBwmpjHbiFipwVopADN99ZJptcdTocuMrU4U+NI/c64DK7/j3g
ceC2Yt4owYf3cuCdofffBXx1ovfqk76iKEoYG7JZpLzTJCLrQ8uaSVyp1Rhz0K4fAloLHFdpz/2U
iFxvxxqBXmNM5ufGPmBeMRctiSd9RVGUKWNyGblHjTEXFtopIo8Ac/LsujP3ksaIiClwmoXGmP0i
chbwWxF5Aegr9gbHokZfURQlhOHURe8YY64stE9EDotImzHmoIi0AUcKnGO/fd0pIo8DrwF+AtSJ
SMQ+7bcD+4u5p5Iw+knf8Pbz2zhvzRu5Z/UaXh5MEneFc2srOff181l+yxuJXnYz200j39l8iIce
fIq9W/fT98oWNvzoDtr9LvxNP+fod37P/j9u59DGI+wYSHJg2GPAC/5z466w5NBTJB9/hgMv7ODo
pr10be+hq3OQ/QmPnlSavpRP0g++jPdWLuBwV4rdvQPs7h5iZ+cg+7qH6OsdZvDYMIn+JIn+flKD
fbRdvISqlnoqmhqyiVhObRN+vBavchZ+ZS1DnmEo6ZPwPKvdxxDXxQ3p+E40RiQWDzT9WBwnGmPP
0UFGQkXVvNB62vNJp318+1pZHSWWo+WP6viZQmux0OKnkjm6feYPITwG4PtpKiM2Ictq+VHHydHx
w7p+scXWMrgZ7X4CLf9kdfexbz/VRdJUxy8RjMFPTkkZhgeB9wB329efjT3ARvQMGWNGRKQJuBT4
gv1l8BhwI/DDQu/Ph2r6iqIoYQz4vl/UcpLcDbxJRLYDV9ptRORCEfmmPWYlsF5ENgKPAXcbY160
+24DPiYiOwg0/m8Vc9GSeNJXFEWZKgxTU2XTGNMFXJFnfD1wq13/A3BOgffvBFZP9rpq9BVFUcIY
cvo4n2mUhNFvW7WQRese5isbDhDjfm6+oI2VN11I8w3v4kjzOfzk5R5+8MArvLx5I0df3sxg5158
L4kbi1P3439i2+9e4OCzh9hxcJADw0FMftpAzBGaK1xaKyIsqIqy/Ytf5ujWLnp297E/4WVj8hNp
n7T1q8ccIe4Kv9x+lJ1Hgpj8oz0JBnqHGRpIMjyYJNnfTXKoDy8xQDo5TOPFF2QbpPhVdfiVtaQr
Z5F0YgymfIYGPRIpQ99Iiv6RNNF4TTYm362wGn5Ix3dj8WwjlGN9I3lj8jOF1oxvSHsevpeksSaW
jcmPR90cHd91JEfPjzpBwTU4PiYfAh0/g0mnqYg4eWPyw9vhRijhc41H0ETl+KJq+XT8E6lDVmxM
/mRzACa6hjKTMVqG4UQQkW+LyBER2RQaKyrtWFEUZdqYXJx+yXE6HbnfBa4eM1Zs2rGiKMq0YIwh
nfSKWkqR02b0jTFPAt1jhq8jSBfGvl6PoijKjMJYSXPipRSZak2/2LRjbDrzGoAFbQUPUxRFObVo
56zTwwRpxxhj1gJrAarnLTMXr/kWffteYuDptfTOX83DO3v44eN72bb5UTp3vMjgkb2kkwncWJzq
5vnMbl9O64I6fvypD2YLqqVNkOhTG804byM0LqylaXkjdcva+dm/PFbQeVsTEapdh4aYS0PM5et/
2JMtqJbPeeuNJPC9ILkp+qq/wq+sJWULqg2mfIaGfRKpFP1Jj75hj74Rj4GkR/+IR6ymPltQLZ/z
1nUdIjGXSNRhoC+Rdd6m00FClp/2s85bk05n76OhpiKnoNrYxbXF0jKdr3wvFfxfFHDeZtfDyVkF
nLfhomkTOXDH7s8kZ43nvD2Rn6xhB+uZ5LzVxloniQGTLmiaSp6pNvpFpR0riqJMFwYzVVU2p4Wp
zsjNpB3DJNKGFUVRpgwDxjdFLaXIaXvSF5EfENSKbhKRfcBnCdKMfyQi7wX2ADedrusriqKcCMZA
OqnJWZPGGHNLgV3HpR1PRKK3h1h/N/MuuIIr1gl7tjxE7+4XGOo6EGjm1bXMmruYhgWLmdNRxyXL
mrn0rEbOaanmf3562CZhRZhbGWFeTYyGpfU0LGmkYeVCapYuIdaxAr+pg42f/lX2mnFXiLsOsyMO
tVGX5gqXWbUVVDXGqWmtZvfmA6QG+3J0/HQqmdXPw7p0f/1iBlOGRMJnKJXM0fAHRjyOjXj0DdlG
KCMe8fo52aSsSNQlEsvo+LYBStTFiThEog5HXukb1fLttTOF0owf6Pm+XW+ZVTGq39tkrKjjEHUl
q+c7jn21hdHG0/HDVEXdnIJoYR1/VHeXgnrzeDq/iIw2UQm93xlzzGQ5ruDaOOc41cXXnFMsvKuO
fwoxRjV9RVGUcsJXo68oilImaMimoihK+WAAv0SdtMWgRl9RFCWMMerInW7mzGvlp9/4COe2VlF7
yftxY3Hi9a3MveAqWhfU8eplTbx+SRMXzZtNx+wo0cPbSL30C/p/vYmrWqtpXFhLw5J6GlYuoG75
IqIdK5C2xaTr2ulJR+gc8tjTM0xt1MlJwKqvjhJvqqKmpYrq1mriLfVUNddR1dZIzzc25iRgjXVE
iuNml61dw/QNB47bvpEgAatvKMXAcLA+MGyduMMeXipNTVNTTgKWE0rKciMSOHdtB6w9m/flJGBl
lnRmOz1aHbN5dsVxCVhRN3DaRp1M16vR9XQqmZ3PRN2uoo6Tk4AVrqiZM17g/ePhWo/teI7bE3W0
FnLequO2fDGanKUoilJGqNFXFEUpJzQjV1EUpXyYoozcYvqLiMgbRWRDaBkWkevtvu+KyK7QvvOK
uW5JPOm3JDqp+MjN/PaZQ7zuY/8nJ/mqLTKMe3ALya2P0f3zrWzfupejW7voOjDA/oTHmn//cDb5
yqubR+eQR+egx+6eIfbsGu1+1dMzzKc76rLJV1UtNVS11FM1p4F4cwNOfQuRxjk4dc348VqG/+Xz
OfcY1vCdaNDpyolEcSIx/vBKT07yVUbDH7Iavpfy8ZLpbLer2saqbPJVpshaJqmqIuIQj0WCbdfh
qf7ubPJVRsMPd7kKluCppaEympN85Y5Zd4RA83dHk7My5NPgw2MRNzf5ypFR/T6ctFXoXOPhkKu9
H5dUNamzhd43zjmPO3aS5z7VGn4Y1fNPL4Ypi9PP9Be5W0Rut9u35dyLMY8B50HwJQHsAH4TOuTv
jTH3T+aiJWH0FUVRpgxj8Kcmeuc6glI1EPQXeZwxRn8MNwK/MsYMncxFVd5RFEUJYUzwpF/McpIU
3V/EcjPwgzFj/yQiz4vIvSJSUcxF9UlfURRlDJPoitUkIutD22ttLxAAROQRYE6e992Zc70J+ovY
UvTnAOtCw3cQfFnECHqP3Ab8w0Q3XBJGf/++Xr6+7yXirvDoVYaRLQ/R/cNtdG3Zx8tbu+g8NMCh
4TRHkx4Dnk8i9A286dXvZFdvgj3bhtjZuY09Rwfp6x1m8Ngwif4kw4ND2cJpF374CuKtzbj1zbj1
LVn93o/X4lfMYsAXBlOGoZSPE4nl1e+daIxILCiW5kRiuBVxHttypKB+7yWD5ifhJigrz59LLOJQ
FXOJRdysfp/R9MONT5KDfcDx+n1Y14egAUp9PJqj30cdJ9vsJF/zk4k0/TAxJ9PgJFe/z/yUzNcA
pVjc0JvGvv1k4ukLvVcl8zLHTOop/qgx5sLCpzJXFtonIpPpL3IT8IAxJhU6d+ZXwoiIfAf4RDE3
rPKOoihKGBunX8xykkymv8gtjJF27BcFEjxRXQ9sKuaiJfGkryiKMlUYpqzgWt7+IiJyIfA+Y8yt
drsDmA88Meb994lIM8GP0w3A+4q5qBp9RVGUMMaQTp5+o2+M6SJPfxFjzHrg1tD2bmBenuMuP5Hr
qtFXFEUJYQz4RsswTCvNsyv45H97HQ0rO7hn9Rp6UmkGPJ+kzYhzJXAk1kQc5lZGaYg5NFdEqGqI
c+v//SND/SOMDA4EDtvBPrzhQXwviTeSyHaXApj1V3eTrpzNQMpnMOWT8HwSKZ++bo++kX4GRmzH
qxGPWXMXWwduDDcWtw7cilBXKzdbHO2VnT2kraPWS6YxxmQ7XY3tcmX8NGfPW5nT3Sq7hIqkRR0H
V8AbHgRyHbaQv8tVfTya12E7tjBaMUlUxxdcy5wj12FbqNPVZBDyO11PpFvW2PMWixZMKy/SavQV
RVHKAwOcwfXW1OgriqKMRZ/0FUVRygTfkJWOz0RKwuj7C87iqb/+Aru6h4hxP4urozTEXGY1VlHV
FKe6tZrqlllUzWmkqqWeWGMDbmMbbn0z2279aVazD5MtjmYTqNxIjPt3JekbORRo97ZAWl8iRSLp
0T/skQglWM1Zviqr2Wcanjiu3bYNTjLJVE+uez5Hs89XIC2cTLWibVZWs4+4wWug5Y+uZ4qlZfwS
Y8k3NrsikqPZZwqkjW1wktGvJ1MYLeJKwSYnJ9uQxB1zglPd4CQ4p2r2yigq7yiKopQJBqPyjqIo
SrmgjlxFUZQyQ43+NLN9z2H+9kP/Cz+VZODptVBdHxRBq5xNKhJnKOWT8Ay9KZ/9yTR9Ix59wykG
kmni9a3HFUJzK4LXSCwa6PE2tv7eB1+0xdByC6D5aZ+05wV6vI2rv+qa87Ox82OLoGVj7F2HqCM8
9P1dOYXQwlp5vrj6pQ3V4xZCCzcrSScTRf0bGj9NTSxQ3ccWQYP8cfWTIVaE7n6icfXhhiynksno
+KrRlw/GaPSOoihK2WDQ6B1FUZSyQTV9RVGUMkPlHUVRlDIh0PSn+y5OHyVh9N1YJS2rLsWNOFyx
TvCS3XipTpsolSbtGXzPz3ajMr4h7Xn4XpI3v/M/WeeqSzzq5nSfGlvQ7NOf/W4oScrP230qwy0X
XIsjHOdozed4He47mn1fMQlPC2qDVpfFdJ+aTAJVdTQ4Uz6f5MkmPEXd3BOcSr+ne5q8qOqcVQqh
T/qKoihlggGmpIXKNKFGX1EUJYTBaPSOoihKuRBE76jRn1bOXtjA77/0NgBqL3n/pN773a+9vehj
P9a5t+hjL50/q+hj8xV8G4+W6tPz31IVPdE2JhMTOR1V0CyqvStTyhnuyD19VmAcRORqEdkmIjtE
5PbpuAdFUZR8ZJ70i1lOBhF5u4hsFhHfNkMvdFxeeykii0TkT3b8P0QkVsx1p9zoi4gLfAV4C7AK
uEVEVk31fSiKohQibYpbTpJNwA3Ak4UOmMBefh641xizBOgB3lvMRafjSX81sMMYs9MYkwR+CFw3
DfehKIpyHD5BGYZilpPBGLPFGLNtgsPy2ksJ4rcvB+63x30PuL6Y64qZYoeFiNwIXG2MudVuvxu4
2BjzwTHHrQHW2M2zCb4VzxSagKMTHlU6nGnzgTNvTuU0n4XGmOYTPbGI/NqevxgqgeHQ9lpjzNpJ
Xu9x4BPGmPV59uW1l8BdwFP2KR8RmQ/8yhhz9kTXm7GOXPsPtxZARNYbYwpqXqWGzmfmc6bNSedT
PMaYq0/VuUTkEWBOnl13GmN+dqquMxmmw+jvB+aHttvtmKIoyhmFMebKkzxFIXvZBdSJSMQY4zEJ
Ozodmv6fgaXW8xwDbgYenIb7UBRFmenktZcm0OUfA260x70HKOqXw5Qbffut9EFgHbAF+JExZvME
b5uURlYC6HxmPmfanHQ+MwwR+c8isg+4BPiliKyz43NF5CGY0F7eBnxMRHYAjcC3irruVDtyFUVR
lOljWpKzFEVRlOlBjb6iKEoZMaONfqmWaxCRb4vIERHZFBprEJGHRWS7fa234yIiX7JzfF5Ezp++
O8+PiMwXkcdE5EWbNv4RO16ScxKRShF5WkQ22vl8zo7nTWsXkQq7vcPu75jO+y+EiLgi8pyI/MJu
l/p8dovICyKyQUTW27GS/MzNJGas0S/xcg3fBcbG+t4OPGqMWQo8archmN9Su6wBvjpF9zgZPODj
xphVwGuBD9j/i1Kd0whwuTHmXOA84GoReS2F09rfC/TY8XvtcTORjxA4+zKU+nwA3miMOS8Uk1+q
n7mZgzFmRi4EHu11oe07gDum+74mcf8dwKbQ9jagza63Advs+teBW/IdN1MXgtCwN50JcwKqgGcJ
shyPAhE7nv38EUROXGLXI/Y4me57HzOPdgIjeDnwC4LmZSU7H3tvu4GmMWMl/5mb7mXGPukD84Bw
reN9dqxUaTXGHLTrh4BWu15S87RSwGuAP1HCc7JSyAbgCPAw8DLQa4IQOci95+x87P4+ghC5mcQX
gU8y2vSpkdKeDwQFL38jIs/YsixQwp+5mcKMLcNwJmOMMSJScrGyIlID/AT4qDHmmIQK3ZfanIwx
aeA8EakDHgBWTPMtnTAi8jbgiDHmGRG5bLrv5xTyF8aY/SLSAjwsIlvDO0vtMzdTmMlP+mdauYbD
ItIGYF+P2PGSmKeIRAkM/n3GmJ/a4ZKeE4Axppcgs/ESbFq73RW+5+x87P5agjT4mcKlwLUispug
CuPlwP+mdOcDgDFmv309QvDFvJoz4DM33cxko3+mlWt4kCBVGnJTph8E/tpGH7wW6Av9fJ0RSPBI
/y1gizHmntCukpyTiDTbJ3xEJE7gn9hC4bT28DxvBH5rrHA8EzDG3GGMaTfGdBD8nfzWGPMuSnQ+
ACJSLSKzMuvAmwkq7ZbkZ25GMd1OhfEW4K3ASwR6653TfT+TuO8fAAeBFIG2+F4CzfRRYDvwCNBg
jxWCKKWXgReAC6f7/vPM5y8I9NXngQ12eWupzgl4NfCcnc8m4DN2/CzgaWAH8GOgwo5X2u0ddv9Z
0z2HceZ2GfCLUp+PvfeNdtmc+fsv1c/cTFq0DIOiKEoZMZPlHUVRFOUUo0ZfURSljFCjryiKUkao
0VcURSkj1OgriqKUEWr0lWlHRNK2kuJmW/ny4yJywp9NEflUaL1DQtVOFaXcUaOvzAQSJqik+CqC
RKm3AJ89ifN9auJDFKU8UaOvzChMkHK/Bvigza50ReSfReTPtk763wGIyGUi8qSI/FKCngtfExFH
RO4G4vaXw332tK6IfMP+kviNzcJVlLJEjb4y4zDG7ARcoIUgm7nPGHMRcBHwtyKyyB66GvgQQb+F
xcANxpjbGf3l8C573FLgK/aXRC/wX6ZuNooys1Cjr8x03kxQU2UDQTnnRgIjDvC0MWanCSpm/oCg
XEQ+dhljNtj1Zwh6HShKWaKllZUZh4icBaQJKigK8CFjzLoxx1xGUA8oTKGaIiOh9TSg8o5StuiT
vjKjEJFm4GvAl01QGGod8N9taWdEZJmtugiw2lZhdYB3AL+z46nM8Yqi5KJP+spMIG7lmyhBP95/
BTIlnL9JIMc8a0s8dwLX231/Br4MLCEoI/yAHV8LPC8izwJ3TsUEFKVU0CqbSkli5Z1PGGPeNt33
oiilhMo7iqIoZYQ+6SuKopQR+qSvKIpSRqjRVxRFKSPU6CuKopQRavQVRVHKCDX6iqIoZcT/B0qy
R2TEdZliAAAAAElFTkSuQmCC
"
>
</div>

</div>

</div>
</div>

</div>
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
</div><div class="inner_cell">
<div class="text_cell_render border-box-sizing rendered_html">
<h3 id="mask&#26500;&#24314;">mask&#26500;&#24314;<a class="anchor-link" href="#mask&#26500;&#24314;">&#182;</a></h3><ol>
<li>padding mask</li>
<li>look ahead</li>
</ol>

</div>
</div>
</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="prompt input_prompt">In&nbsp;[&nbsp;]:</div>
<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="c1"># 1. padding mask</span>
<span class="c1"># batch_data.shape: [batch_size, seq_len]</span>
<span class="k">def</span> <span class="nf">create_padding_mask</span><span class="p">(</span><span class="n">batch_data</span><span class="p">):</span>
    <span class="n">padding_mask</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">cast</span><span class="p">(</span><span class="n">tf</span><span class="o">.</span><span class="n">math</span><span class="o">.</span><span class="n">equal</span><span class="p">(</span><span class="n">batch_data</span><span class="p">,</span> <span class="mi">0</span><span class="p">),</span> <span class="n">tf</span><span class="o">.</span><span class="n">float32</span><span class="p">)</span>
    <span class="c1"># [batch_size, 1, 1, seq_len]</span>
    <span class="k">return</span> <span class="n">padding_mask</span><span class="p">[:,</span> <span class="n">tf</span><span class="o">.</span><span class="n">newaxis</span><span class="p">,</span> <span class="n">tf</span><span class="o">.</span><span class="n">newaxis</span><span class="p">,</span> <span class="p">:]</span>
</pre></div>

    </div>
</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="prompt input_prompt">In&nbsp;[21]:</div>
<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="c1"># test</span>
<span class="n">x</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">constant</span><span class="p">([[</span><span class="mi">7</span><span class="p">,</span> <span class="mi">6</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">],</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">0</span><span class="p">],</span> <span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">5</span><span class="p">]])</span>
<span class="n">create_padding_mask</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
</pre></div>

    </div>
</div>
</div>

<div class="output_wrapper">
<div class="output">


<div class="output_area">

    <div class="prompt output_prompt">Out[21]:</div>




<div class="output_text output_subarea output_execute_result">
<pre>&lt;tf.Tensor: shape=(3, 1, 1, 5), dtype=float32, numpy=
array([[[[0., 0., 1., 1., 0.]]],


       [[[0., 0., 0., 1., 1.]]],


       [[[1., 1., 1., 0., 0.]]]], dtype=float32)&gt;</pre>
</div>

</div>

</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="prompt input_prompt">In&nbsp;[&nbsp;]:</div>
<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="c1"># 2. look ahead只能看到前面的，看不到后面的</span>
<span class="c1"># attention_weights.shape: [3, 3]</span>
<span class="c1"># [[1,0,0]</span>
<span class="c1">#  [4,5,0]</span>
<span class="c1">#  [7,8,9]]</span>
<span class="c1"># 下三角都是0，上三角都是1</span>
<span class="k">def</span> <span class="nf">create_look_ahead_mask</span><span class="p">(</span><span class="n">size</span><span class="p">):</span>
    <span class="n">mask</span> <span class="o">=</span> <span class="mi">1</span> <span class="o">-</span> <span class="n">tf</span><span class="o">.</span><span class="n">linalg</span><span class="o">.</span><span class="n">band_part</span><span class="p">(</span><span class="n">tf</span><span class="o">.</span><span class="n">ones</span><span class="p">((</span><span class="n">size</span><span class="p">,</span> <span class="n">size</span><span class="p">)),</span> <span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">0</span><span class="p">)</span>
    <span class="c1"># shape: [seq_len, seq_len]</span>
    <span class="k">return</span> <span class="n">mask</span>
</pre></div>

    </div>
</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="prompt input_prompt">In&nbsp;[23]:</div>
<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="c1"># test </span>
<span class="n">create_look_ahead_mask</span><span class="p">(</span><span class="mi">3</span><span class="p">)</span>
</pre></div>

    </div>
</div>
</div>

<div class="output_wrapper">
<div class="output">


<div class="output_area">

    <div class="prompt output_prompt">Out[23]:</div>




<div class="output_text output_subarea output_execute_result">
<pre>&lt;tf.Tensor: shape=(3, 3), dtype=float32, numpy=
array([[0., 1., 1.],
       [0., 0., 1.],
       [0., 0., 0.]], dtype=float32)&gt;</pre>
</div>

</div>

</div>
</div>

</div>
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
</div><div class="inner_cell">
<div class="text_cell_render border-box-sizing rendered_html">
<h3 id="&#32553;&#25918;&#28857;&#31215;&#27880;&#24847;&#21147;&#26426;&#21046;">&#32553;&#25918;&#28857;&#31215;&#27880;&#24847;&#21147;&#26426;&#21046;<a class="anchor-link" href="#&#32553;&#25918;&#28857;&#31215;&#27880;&#24847;&#21147;&#26426;&#21046;">&#182;</a></h3>
</div>
</div>
</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="prompt input_prompt">In&nbsp;[&nbsp;]:</div>
<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="k">def</span> <span class="nf">scaled_dot_product_attention</span><span class="p">(</span><span class="n">q</span><span class="p">,</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span><span class="p">,</span> <span class="n">mask</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Args:</span>
<span class="sd">    - q: shape == (..., seq_len_q, depth)</span>
<span class="sd">    - k: shape == (..., seq_len_k, depth)</span>
<span class="sd">    - v: shape == (..., seq_len_v, depth_v)</span>
<span class="sd">    - seq_len_k == seq_len_v</span>
<span class="sd">    - mask: shape == (..., seq_len_q, seq_len_k) 默认为None</span>
<span class="sd">    Returns:</span>
<span class="sd">    - output: weighted sum</span>
<span class="sd">    - attention_weights: weights of attention</span>
<span class="sd">    &quot;&quot;&quot;</span>
    
    <span class="c1"># matmul_qk.hape: (..., seq_len_q, seq_len_k)</span>
    <span class="n">matmul_qk</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">matmul</span><span class="p">(</span><span class="n">q</span><span class="p">,</span> <span class="n">k</span><span class="p">,</span> <span class="n">transpose_b</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
    <span class="n">dk</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">cast</span><span class="p">(</span><span class="n">tf</span><span class="o">.</span><span class="n">shape</span><span class="p">(</span><span class="n">k</span><span class="p">)[</span><span class="o">-</span><span class="mi">1</span><span class="p">],</span> <span class="n">tf</span><span class="o">.</span><span class="n">float32</span><span class="p">)</span>
    <span class="n">scaled_attention_logits</span> <span class="o">=</span> <span class="n">matmul_qk</span> <span class="o">/</span> <span class="n">tf</span><span class="o">.</span><span class="n">math</span><span class="o">.</span><span class="n">sqrt</span><span class="p">(</span><span class="n">dk</span><span class="p">)</span>
    
    <span class="k">if</span> <span class="n">mask</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
        <span class="c1"># 目的是使得在softmax后，值趋近于0</span>
        <span class="n">scaled_attention_logits</span> <span class="o">+=</span> <span class="p">(</span><span class="n">mask</span> <span class="o">*</span> <span class="o">-</span><span class="mf">1e9</span><span class="p">)</span>
    
    <span class="c1"># attention_weights.shape: [..., seq_len_q, seq_len_k]</span>
    <span class="n">attention_weights</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">nn</span><span class="o">.</span><span class="n">softmax</span><span class="p">(</span><span class="n">scaled_attention_logits</span><span class="p">,</span> <span class="n">axis</span> <span class="o">=</span> <span class="o">-</span><span class="mi">1</span><span class="p">)</span>
    
    <span class="c1"># 加权求和</span>
    <span class="c1"># output.hape: (..., seq_len_q, depth_v)</span>
    <span class="n">output</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">matmul</span><span class="p">(</span><span class="n">attention_weights</span><span class="p">,</span> <span class="n">v</span><span class="p">)</span>
    <span class="k">return</span> <span class="n">output</span><span class="p">,</span> <span class="n">attention_weights</span>
</pre></div>

    </div>
</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="prompt input_prompt">In&nbsp;[&nbsp;]:</div>
<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="c1"># 打印缩放点积，方便调试</span>
<span class="k">def</span> <span class="nf">print_scaled_dot_product_attention</span><span class="p">(</span><span class="n">q</span><span class="p">,</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span><span class="p">):</span>
    <span class="n">temp_out</span><span class="p">,</span> <span class="n">temp_att</span> <span class="o">=</span> <span class="n">scaled_dot_product_attention</span><span class="p">(</span><span class="n">q</span><span class="p">,</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
    <span class="nb">print</span><span class="p">(</span><span class="s1">&#39;Attention weights are:&#39;</span><span class="p">)</span>
    <span class="nb">print</span><span class="p">(</span><span class="n">temp_att</span><span class="p">)</span>
    <span class="nb">print</span><span class="p">(</span><span class="s1">&#39;Output is:&#39;</span><span class="p">)</span>
    <span class="nb">print</span><span class="p">(</span><span class="n">temp_out</span><span class="p">)</span>
</pre></div>

    </div>
</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="prompt input_prompt">In&nbsp;[26]:</div>
<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="c1"># 小数四舍五入</span>
<span class="n">np</span><span class="o">.</span><span class="n">set_printoptions</span><span class="p">(</span><span class="n">suppress</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="c1"># test</span>
<span class="n">temp_k</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">constant</span><span class="p">([[</span><span class="mi">10</span><span class="p">,</span><span class="mi">0</span><span class="p">,</span><span class="mi">0</span><span class="p">],</span>
                      <span class="p">[</span><span class="mi">0</span><span class="p">,</span><span class="mi">10</span><span class="p">,</span><span class="mi">0</span><span class="p">],</span>
                      <span class="p">[</span><span class="mi">0</span><span class="p">,</span><span class="mi">0</span><span class="p">,</span><span class="mi">10</span><span class="p">],</span>
                      <span class="p">[</span><span class="mi">0</span><span class="p">,</span><span class="mi">0</span><span class="p">,</span><span class="mi">10</span><span class="p">]],</span> <span class="n">dtype</span><span class="o">=</span><span class="n">tf</span><span class="o">.</span><span class="n">float32</span><span class="p">)</span>  <span class="c1"># (4, 3)</span>

<span class="n">temp_v</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">constant</span><span class="p">([[</span>   <span class="mi">1</span><span class="p">,</span><span class="mi">0</span><span class="p">],</span>
                      <span class="p">[</span>  <span class="mi">10</span><span class="p">,</span><span class="mi">0</span><span class="p">],</span>
                      <span class="p">[</span> <span class="mi">100</span><span class="p">,</span><span class="mi">5</span><span class="p">],</span>
                      <span class="p">[</span><span class="mi">1000</span><span class="p">,</span><span class="mi">6</span><span class="p">]],</span> <span class="n">dtype</span><span class="o">=</span><span class="n">tf</span><span class="o">.</span><span class="n">float32</span><span class="p">)</span>  <span class="c1"># (4, 2)</span>

<span class="n">temp_q1</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">constant</span><span class="p">([[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">10</span><span class="p">,</span> <span class="mi">0</span><span class="p">]],</span> <span class="n">dtype</span><span class="o">=</span><span class="n">tf</span><span class="o">.</span><span class="n">float32</span><span class="p">)</span>  <span class="c1"># (1, 3)</span>
<span class="n">print_scaled_dot_product_attention</span><span class="p">(</span><span class="n">temp_q1</span><span class="p">,</span> <span class="n">temp_k</span><span class="p">,</span> <span class="n">temp_v</span><span class="p">)</span>
</pre></div>

    </div>
</div>
</div>

<div class="output_wrapper">
<div class="output">


<div class="output_area">

    <div class="prompt"></div>


<div class="output_subarea output_stream output_stdout output_text">
<pre>Attention weights are:
tf.Tensor([[0. 1. 0. 0.]], shape=(1, 4), dtype=float32)
Output is:
tf.Tensor([[10.  0.]], shape=(1, 2), dtype=float32)
</pre>
</div>
</div>

</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="prompt input_prompt">In&nbsp;[27]:</div>
<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="n">temp_q2</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">constant</span><span class="p">([[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">10</span><span class="p">]],</span> <span class="n">dtype</span><span class="o">=</span><span class="n">tf</span><span class="o">.</span><span class="n">float32</span><span class="p">)</span>  <span class="c1"># (1, 3)</span>
<span class="n">print_scaled_dot_product_attention</span><span class="p">(</span><span class="n">temp_q2</span><span class="p">,</span> <span class="n">temp_k</span><span class="p">,</span> <span class="n">temp_v</span><span class="p">)</span>
</pre></div>

    </div>
</div>
</div>

<div class="output_wrapper">
<div class="output">


<div class="output_area">

    <div class="prompt"></div>


<div class="output_subarea output_stream output_stdout output_text">
<pre>Attention weights are:
tf.Tensor([[0.  0.  0.5 0.5]], shape=(1, 4), dtype=float32)
Output is:
tf.Tensor([[550.    5.5]], shape=(1, 2), dtype=float32)
</pre>
</div>
</div>

</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="prompt input_prompt">In&nbsp;[28]:</div>
<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="n">temp_q3</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">constant</span><span class="p">([[</span><span class="mi">10</span><span class="p">,</span> <span class="mi">10</span><span class="p">,</span> <span class="mi">0</span><span class="p">]],</span> <span class="n">dtype</span><span class="o">=</span><span class="n">tf</span><span class="o">.</span><span class="n">float32</span><span class="p">)</span>  <span class="c1"># (1, 3)</span>
<span class="n">print_scaled_dot_product_attention</span><span class="p">(</span><span class="n">temp_q3</span><span class="p">,</span> <span class="n">temp_k</span><span class="p">,</span> <span class="n">temp_v</span><span class="p">)</span>
</pre></div>

    </div>
</div>
</div>

<div class="output_wrapper">
<div class="output">


<div class="output_area">

    <div class="prompt"></div>


<div class="output_subarea output_stream output_stdout output_text">
<pre>Attention weights are:
tf.Tensor([[0.5 0.5 0.  0. ]], shape=(1, 4), dtype=float32)
Output is:
tf.Tensor([[5.5 0. ]], shape=(1, 2), dtype=float32)
</pre>
</div>
</div>

</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="prompt input_prompt">In&nbsp;[29]:</div>
<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="n">temp_q4</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">constant</span><span class="p">([[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">10</span><span class="p">],</span> <span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">10</span><span class="p">,</span> <span class="mi">0</span><span class="p">],</span> <span class="p">[</span><span class="mi">10</span><span class="p">,</span> <span class="mi">10</span><span class="p">,</span> <span class="mi">0</span><span class="p">]],</span> <span class="n">dtype</span><span class="o">=</span><span class="n">tf</span><span class="o">.</span><span class="n">float32</span><span class="p">)</span>  <span class="c1"># (3, 3)</span>
<span class="n">print_scaled_dot_product_attention</span><span class="p">(</span><span class="n">temp_q4</span><span class="p">,</span> <span class="n">temp_k</span><span class="p">,</span> <span class="n">temp_v</span><span class="p">)</span>
</pre></div>

    </div>
</div>
</div>

<div class="output_wrapper">
<div class="output">


<div class="output_area">

    <div class="prompt"></div>


<div class="output_subarea output_stream output_stdout output_text">
<pre>Attention weights are:
tf.Tensor(
[[0.  0.  0.5 0.5]
 [0.  1.  0.  0. ]
 [0.5 0.5 0.  0. ]], shape=(3, 4), dtype=float32)
Output is:
tf.Tensor(
[[550.    5.5]
 [ 10.    0. ]
 [  5.5   0. ]], shape=(3, 2), dtype=float32)
</pre>
</div>
</div>

</div>
</div>

</div>
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
</div><div class="inner_cell">
<div class="text_cell_render border-box-sizing rendered_html">
<h2 id="&#27169;&#22411;&#26500;&#24314;">&#27169;&#22411;&#26500;&#24314;<a class="anchor-link" href="#&#27169;&#22411;&#26500;&#24314;">&#182;</a></h2><h3 id="&#22810;&#22836;&#27880;&#24847;&#21147;">&#22810;&#22836;&#27880;&#24847;&#21147;<a class="anchor-link" href="#&#22810;&#22836;&#27880;&#24847;&#21147;">&#182;</a></h3>
</div>
</div>
</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="prompt input_prompt">In&nbsp;[&nbsp;]:</div>
<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="k">class</span> <span class="nc">MultiHeadAttention</span><span class="p">(</span><span class="n">keras</span><span class="o">.</span><span class="n">layers</span><span class="o">.</span><span class="n">Layer</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    理论上：</span>
<span class="sd">    x -&gt; Wq0 -&gt; q0</span>
<span class="sd">    x -&gt; Wk0 -&gt; k0</span>
<span class="sd">    x -&gt; Wv0 -&gt; v0</span>
<span class="sd">    实际上：把x分成q,k,v</span>
<span class="sd">    q -&gt; Wq0 -&gt; q0</span>
<span class="sd">    k -&gt; Wk0 -&gt; k0</span>
<span class="sd">    v -&gt; Wv0 -&gt; v0</span>
<span class="sd">    实战中的技巧：</span>
<span class="sd">    q -&gt; Wq -&gt; Q -&gt; split -&gt; q0,q1,q2...</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">d_model</span><span class="p">,</span> <span class="n">num_heads</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">MultiHeadAttention</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">num_heads</span> <span class="o">=</span> <span class="n">num_heads</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">d_model</span> <span class="o">=</span> <span class="n">d_model</span>
        <span class="k">assert</span> <span class="bp">self</span><span class="o">.</span><span class="n">d_model</span> <span class="o">%</span> <span class="bp">self</span><span class="o">.</span><span class="n">num_heads</span> <span class="o">==</span> <span class="mi">0</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">depth</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">d_model</span> <span class="o">//</span> <span class="bp">self</span><span class="o">.</span><span class="n">num_heads</span>
        
        <span class="bp">self</span><span class="o">.</span><span class="n">WQ</span> <span class="o">=</span> <span class="n">keras</span><span class="o">.</span><span class="n">layers</span><span class="o">.</span><span class="n">Dense</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">d_model</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">WK</span> <span class="o">=</span> <span class="n">keras</span><span class="o">.</span><span class="n">layers</span><span class="o">.</span><span class="n">Dense</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">d_model</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">WV</span> <span class="o">=</span> <span class="n">keras</span><span class="o">.</span><span class="n">layers</span><span class="o">.</span><span class="n">Dense</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">d_model</span><span class="p">)</span>
        
        <span class="bp">self</span><span class="o">.</span><span class="n">dense</span> <span class="o">=</span> <span class="n">keras</span><span class="o">.</span><span class="n">layers</span><span class="o">.</span><span class="n">Dense</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">d_model</span><span class="p">)</span>
        
    <span class="k">def</span> <span class="nf">split_heads</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">x</span><span class="p">,</span> <span class="n">batch_size</span><span class="p">):</span>
        <span class="c1"># x.shape: (batch_size, seq_len, d_model)</span>
        <span class="c1"># d_model = num_heads * depth</span>
        <span class="c1"># x -&gt; (batch_size, num_heads, seq_len, depth)</span>
        <span class="n">x</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">reshape</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="p">(</span><span class="n">batch_size</span><span class="p">,</span> <span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">num_heads</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">depth</span><span class="p">))</span>
        <span class="k">return</span> <span class="n">tf</span><span class="o">.</span><span class="n">transpose</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">perm</span><span class="o">=</span><span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">3</span><span class="p">])</span>
    
    <span class="k">def</span> <span class="nf">call</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">q</span><span class="p">,</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span><span class="p">,</span> <span class="n">mask</span><span class="p">):</span>
        <span class="n">batch_size</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">shape</span><span class="p">(</span><span class="n">q</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
        
        <span class="n">q</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">WQ</span><span class="p">(</span><span class="n">q</span><span class="p">)</span> <span class="c1"># q.shape: (batch_size, seq_len_q, d_model)</span>
        <span class="n">k</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">WK</span><span class="p">(</span><span class="n">k</span><span class="p">)</span> <span class="c1"># k.shape: (batch_size, seq_len_k, d_model)</span>
        <span class="n">v</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">WV</span><span class="p">(</span><span class="n">v</span><span class="p">)</span> <span class="c1"># v.shape: (batch_size, seq_len_v, d_model)</span>
        
        <span class="n">q</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">split_heads</span><span class="p">(</span><span class="n">q</span><span class="p">,</span> <span class="n">batch_size</span><span class="p">)</span> <span class="c1"># q.shape: (batch_size, num_heads, seq_len_q, depth)</span>
        <span class="n">k</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">split_heads</span><span class="p">(</span><span class="n">k</span><span class="p">,</span> <span class="n">batch_size</span><span class="p">)</span> <span class="c1"># k.shape: (batch_size, num_heads, seq_len_k, depth)</span>
        <span class="n">v</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">split_heads</span><span class="p">(</span><span class="n">v</span><span class="p">,</span> <span class="n">batch_size</span><span class="p">)</span> <span class="c1"># v.shape: (batch_size, num_heads, seq_len_v, depth)</span>
        
        <span class="c1"># scaled_attention_outputs.shape: (batch_size, num_heads, seq_len_q, depth)</span>
        <span class="c1"># attention_weights.shape: (batch_size, num_heads, seq_len_q, seq_len_k)</span>
        <span class="n">scaled_attention_outputs</span><span class="p">,</span> <span class="n">attention_weights</span> <span class="o">=</span> <span class="n">scaled_dot_product_attention</span><span class="p">(</span><span class="n">q</span><span class="p">,</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span><span class="p">,</span> <span class="n">mask</span><span class="p">)</span>
        
        <span class="c1"># scaled_attention_outputs.shape: (batch_size, seq_len_q, num_heads, depth)</span>
        <span class="n">scaled_attention_outputs</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">transpose</span><span class="p">(</span><span class="n">scaled_attention_outputs</span><span class="p">,</span> <span class="n">perm</span><span class="o">=</span><span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">3</span><span class="p">])</span>
        
        <span class="c1"># concat_attention.shape: (batch_size, seq_len_q, d_model)</span>
        <span class="n">concat_attention</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">reshape</span><span class="p">(</span><span class="n">scaled_attention_outputs</span><span class="p">,</span> <span class="p">(</span><span class="n">batch_size</span><span class="p">,</span> <span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">d_model</span><span class="p">))</span>
        
        <span class="c1"># output.shape: (batch_size, seq_len_q, d_model)</span>
        <span class="n">output</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">dense</span><span class="p">(</span><span class="n">concat_attention</span><span class="p">)</span>
        <span class="k">return</span> <span class="n">output</span><span class="p">,</span> <span class="n">attention_weights</span>
</pre></div>

    </div>
</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="prompt input_prompt">In&nbsp;[31]:</div>
<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="c1"># test</span>
<span class="n">temp_mha</span> <span class="o">=</span> <span class="n">MultiHeadAttention</span><span class="p">(</span><span class="n">d_model</span><span class="o">=</span><span class="mi">512</span><span class="p">,</span> <span class="n">num_heads</span><span class="o">=</span><span class="mi">8</span><span class="p">)</span>
<span class="n">y</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">uniform</span><span class="p">((</span><span class="mi">1</span><span class="p">,</span> <span class="mi">60</span><span class="p">,</span> <span class="mi">512</span><span class="p">))</span>  <span class="c1"># (batch_size, seq_len_q, d_model)</span>
<span class="n">out</span><span class="p">,</span> <span class="n">attn</span> <span class="o">=</span> <span class="n">temp_mha</span><span class="p">(</span><span class="n">y</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">mask</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span>
<span class="n">out</span><span class="o">.</span><span class="n">shape</span><span class="p">,</span> <span class="n">attn</span><span class="o">.</span><span class="n">shape</span>
</pre></div>

    </div>
</div>
</div>

<div class="output_wrapper">
<div class="output">


<div class="output_area">

    <div class="prompt output_prompt">Out[31]:</div>




<div class="output_text output_subarea output_execute_result">
<pre>(TensorShape([1, 60, 512]), TensorShape([1, 8, 60, 60]))</pre>
</div>

</div>

</div>
</div>

</div>
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
</div><div class="inner_cell">
<div class="text_cell_render border-box-sizing rendered_html">
<h3 id="feed_forward">feed_forward<a class="anchor-link" href="#feed_forward">&#182;</a></h3>
</div>
</div>
</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="prompt input_prompt">In&nbsp;[&nbsp;]:</div>
<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="k">def</span> <span class="nf">feed_forward_network</span><span class="p">(</span><span class="n">d_model</span><span class="p">,</span> <span class="n">dff</span><span class="p">):</span>
    <span class="c1"># dff: dim of feed forward network.</span>
    <span class="k">return</span> <span class="n">keras</span><span class="o">.</span><span class="n">Sequential</span><span class="p">([</span>
        <span class="n">keras</span><span class="o">.</span><span class="n">layers</span><span class="o">.</span><span class="n">Dense</span><span class="p">(</span><span class="n">dff</span><span class="p">,</span> <span class="n">activation</span><span class="o">=</span><span class="s1">&#39;relu&#39;</span><span class="p">),</span>
        <span class="n">keras</span><span class="o">.</span><span class="n">layers</span><span class="o">.</span><span class="n">Dense</span><span class="p">(</span><span class="n">d_model</span><span class="p">)</span>
    <span class="p">])</span>
</pre></div>

    </div>
</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="prompt input_prompt">In&nbsp;[33]:</div>
<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="c1"># test</span>
<span class="n">sample_ffn</span> <span class="o">=</span> <span class="n">feed_forward_network</span><span class="p">(</span><span class="mi">512</span><span class="p">,</span> <span class="mi">2048</span><span class="p">)</span>
<span class="n">sample_ffn</span><span class="p">(</span><span class="n">tf</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">uniform</span><span class="p">((</span><span class="mi">64</span><span class="p">,</span> <span class="mi">50</span><span class="p">,</span> <span class="mi">512</span><span class="p">)))</span><span class="o">.</span><span class="n">shape</span>
</pre></div>

    </div>
</div>
</div>

<div class="output_wrapper">
<div class="output">


<div class="output_area">

    <div class="prompt output_prompt">Out[33]:</div>




<div class="output_text output_subarea output_execute_result">
<pre>TensorShape([64, 50, 512])</pre>
</div>

</div>

</div>
</div>

</div>
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
</div><div class="inner_cell">
<div class="text_cell_render border-box-sizing rendered_html">
<h3 id="Encoder-Layer">Encoder Layer<a class="anchor-link" href="#Encoder-Layer">&#182;</a></h3>
</div>
</div>
</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="prompt input_prompt">In&nbsp;[&nbsp;]:</div>
<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="k">class</span> <span class="nc">EncoderLayer</span><span class="p">(</span><span class="n">keras</span><span class="o">.</span><span class="n">layers</span><span class="o">.</span><span class="n">Layer</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    x -&gt; self attention -&gt; add &amp; normalize &amp; dropout -&gt; feed_forward -&gt; add &amp; normalize &amp; dropout</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">d_model</span><span class="p">,</span> <span class="n">num_heads</span><span class="p">,</span> <span class="n">dff</span><span class="p">,</span> <span class="n">rate</span> <span class="o">=</span> <span class="mf">0.1</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">EncoderLayer</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">mha</span> <span class="o">=</span> <span class="n">MultiHeadAttention</span><span class="p">(</span><span class="n">d_model</span><span class="p">,</span> <span class="n">num_heads</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">ffn</span> <span class="o">=</span> <span class="n">feed_forward_network</span><span class="p">(</span><span class="n">d_model</span><span class="p">,</span> <span class="n">dff</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">layer_norm1</span> <span class="o">=</span> <span class="n">keras</span><span class="o">.</span><span class="n">layers</span><span class="o">.</span><span class="n">LayerNormalization</span><span class="p">(</span><span class="n">epsilon</span><span class="o">=</span><span class="mf">1e-6</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">layer_norm2</span> <span class="o">=</span> <span class="n">keras</span><span class="o">.</span><span class="n">layers</span><span class="o">.</span><span class="n">LayerNormalization</span><span class="p">(</span><span class="n">epsilon</span><span class="o">=</span><span class="mf">1e-6</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">dropout1</span> <span class="o">=</span> <span class="n">keras</span><span class="o">.</span><span class="n">layers</span><span class="o">.</span><span class="n">Dropout</span><span class="p">(</span><span class="n">rate</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">dropout2</span> <span class="o">=</span> <span class="n">keras</span><span class="o">.</span><span class="n">layers</span><span class="o">.</span><span class="n">Dropout</span><span class="p">(</span><span class="n">rate</span><span class="p">)</span>
        
    <span class="k">def</span> <span class="nf">call</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">x</span><span class="p">,</span> <span class="n">training</span><span class="p">,</span> <span class="n">encoder_padding_mask</span><span class="p">):</span>
        <span class="c1"># x.shape: (batch_size, seq_len, dim=d_model)</span>
        <span class="c1"># attn_output.shape: (batch_size, seq_len, d_model)</span>
        <span class="c1"># out1.shape: (batch_size, seq_len, d_model)</span>
        <span class="n">attn_output</span><span class="p">,</span> <span class="n">_</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">mha</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">x</span><span class="p">,</span> <span class="n">x</span><span class="p">,</span> <span class="n">encoder_padding_mask</span><span class="p">)</span> 
        <span class="n">attn_output</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">dropout1</span><span class="p">(</span><span class="n">attn_output</span><span class="p">,</span> <span class="n">training</span><span class="o">=</span><span class="n">training</span><span class="p">)</span>
        <span class="n">out1</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">layer_norm1</span><span class="p">(</span><span class="n">x</span> <span class="o">+</span> <span class="n">attn_output</span><span class="p">)</span>
        
        <span class="c1"># ffn_output.shape: (batch_size, seq_len, d_model)</span>
        <span class="c1"># out2.shape: (batch_size, seq_len, d_model)</span>
        <span class="n">ffn_output</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">ffn</span><span class="p">(</span><span class="n">out1</span><span class="p">)</span>
        <span class="n">ffn_output</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">dropout2</span><span class="p">(</span><span class="n">ffn_output</span><span class="p">,</span> <span class="n">training</span><span class="o">=</span><span class="n">training</span><span class="p">)</span>
        <span class="n">out2</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">layer_norm2</span><span class="p">(</span><span class="n">out1</span> <span class="o">+</span> <span class="n">ffn_output</span><span class="p">)</span>
        <span class="k">return</span> <span class="n">out2</span>
</pre></div>

    </div>
</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="prompt input_prompt">In&nbsp;[35]:</div>
<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="c1"># test</span>
<span class="n">sample_encoder_layer</span> <span class="o">=</span> <span class="n">EncoderLayer</span><span class="p">(</span><span class="mi">512</span><span class="p">,</span> <span class="mi">8</span><span class="p">,</span> <span class="mi">2048</span><span class="p">)</span>
<span class="n">sample_input</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">uniform</span><span class="p">((</span><span class="mi">64</span><span class="p">,</span> <span class="mi">50</span><span class="p">,</span> <span class="mi">512</span><span class="p">))</span>
<span class="n">sample_output</span> <span class="o">=</span> <span class="n">sample_encoder_layer</span><span class="p">(</span><span class="n">sample_input</span><span class="p">,</span> <span class="kc">False</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>

<span class="n">sample_output</span><span class="o">.</span><span class="n">shape</span>  <span class="c1"># (batch_size, input_seq_len, d_model)</span>
</pre></div>

    </div>
</div>
</div>

<div class="output_wrapper">
<div class="output">


<div class="output_area">

    <div class="prompt output_prompt">Out[35]:</div>




<div class="output_text output_subarea output_execute_result">
<pre>TensorShape([64, 50, 512])</pre>
</div>

</div>

</div>
</div>

</div>
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
</div><div class="inner_cell">
<div class="text_cell_render border-box-sizing rendered_html">
<h3 id="Decoder-Layer">Decoder Layer<a class="anchor-link" href="#Decoder-Layer">&#182;</a></h3>
</div>
</div>
</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="prompt input_prompt">In&nbsp;[&nbsp;]:</div>
<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="k">class</span> <span class="nc">DecoderLayer</span><span class="p">(</span><span class="n">keras</span><span class="o">.</span><span class="n">layers</span><span class="o">.</span><span class="n">Layer</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    x -&gt; self attention -&gt; add &amp; normalize &amp; dropout -&gt; out1</span>
<span class="sd">    out1, encoding_outputs -&gt; attention -&gt; add &amp; normalize &amp; dropout -&gt; out2</span>
<span class="sd">    out2 -&gt; feed_forward -&gt; add &amp; normalize &amp; dropout -&gt; out3</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">d_model</span><span class="p">,</span> <span class="n">num_heads</span><span class="p">,</span> <span class="n">dff</span><span class="p">,</span> <span class="n">rate</span> <span class="o">=</span> <span class="mf">0.1</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">DecoderLayer</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">mha1</span> <span class="o">=</span> <span class="n">MultiHeadAttention</span><span class="p">(</span><span class="n">d_model</span><span class="p">,</span> <span class="n">num_heads</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">mha2</span> <span class="o">=</span> <span class="n">MultiHeadAttention</span><span class="p">(</span><span class="n">d_model</span><span class="p">,</span> <span class="n">num_heads</span><span class="p">)</span>
        
        <span class="bp">self</span><span class="o">.</span><span class="n">ffn</span> <span class="o">=</span> <span class="n">feed_forward_network</span><span class="p">(</span><span class="n">d_model</span><span class="p">,</span> <span class="n">dff</span><span class="p">)</span>
        
        <span class="bp">self</span><span class="o">.</span><span class="n">layer_norm1</span> <span class="o">=</span> <span class="n">keras</span><span class="o">.</span><span class="n">layers</span><span class="o">.</span><span class="n">LayerNormalization</span><span class="p">(</span><span class="n">epsilon</span><span class="o">=</span><span class="mf">1e-6</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">layer_norm2</span> <span class="o">=</span> <span class="n">keras</span><span class="o">.</span><span class="n">layers</span><span class="o">.</span><span class="n">LayerNormalization</span><span class="p">(</span><span class="n">epsilon</span><span class="o">=</span><span class="mf">1e-6</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">layer_norm3</span> <span class="o">=</span> <span class="n">keras</span><span class="o">.</span><span class="n">layers</span><span class="o">.</span><span class="n">LayerNormalization</span><span class="p">(</span><span class="n">epsilon</span><span class="o">=</span><span class="mf">1e-6</span><span class="p">)</span>
        
        <span class="bp">self</span><span class="o">.</span><span class="n">dropout1</span> <span class="o">=</span> <span class="n">keras</span><span class="o">.</span><span class="n">layers</span><span class="o">.</span><span class="n">Dropout</span><span class="p">(</span><span class="n">rate</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">dropout2</span> <span class="o">=</span> <span class="n">keras</span><span class="o">.</span><span class="n">layers</span><span class="o">.</span><span class="n">Dropout</span><span class="p">(</span><span class="n">rate</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">dropout3</span> <span class="o">=</span> <span class="n">keras</span><span class="o">.</span><span class="n">layers</span><span class="o">.</span><span class="n">Dropout</span><span class="p">(</span><span class="n">rate</span><span class="p">)</span>
        
    <span class="k">def</span> <span class="nf">call</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">x</span><span class="p">,</span> <span class="n">encoding_outputs</span><span class="p">,</span> <span class="n">training</span><span class="p">,</span> <span class="n">decoder_mask</span><span class="p">,</span> <span class="n">encoder_decoder_padding_mask</span><span class="p">):</span>
        <span class="c1"># decoder_mask是由look_ahead_mask和decoder_padding_mask做与操作合并而来</span>
        <span class="c1"># x.shape: (batch_size, target_seq_len, d_model)</span>
        <span class="c1"># encoding_outputs.shape: (batch_size, input_seq_len, d_model)</span>
        <span class="c1"># attn1,out1.shape: (batch_size, target_seq_len, d_model)</span>
        <span class="n">attn1</span><span class="p">,</span> <span class="n">attn_weights1</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">mha1</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">x</span><span class="p">,</span> <span class="n">x</span><span class="p">,</span> <span class="n">decoder_mask</span><span class="p">)</span> 
        <span class="n">attn1</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">dropout1</span><span class="p">(</span><span class="n">attn1</span><span class="p">,</span> <span class="n">training</span> <span class="o">=</span> <span class="n">training</span><span class="p">)</span>
        <span class="n">out1</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">layer_norm1</span><span class="p">(</span><span class="n">x</span> <span class="o">+</span> <span class="n">attn1</span><span class="p">)</span>
        
        <span class="c1"># attn2,out2.shape: (batch_size, target_seq_len, d_model)</span>
        <span class="n">attn2</span><span class="p">,</span> <span class="n">attn_weights2</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">mha2</span><span class="p">(</span>
            <span class="n">out1</span><span class="p">,</span> <span class="n">encoding_outputs</span><span class="p">,</span> <span class="n">encoding_outputs</span><span class="p">,</span> <span class="n">encoder_decoder_padding_mask</span><span class="p">)</span> 
        <span class="n">attn2</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">dropout2</span><span class="p">(</span><span class="n">attn2</span><span class="p">,</span> <span class="n">training</span> <span class="o">=</span> <span class="n">training</span><span class="p">)</span>
        <span class="n">out2</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">layer_norm2</span><span class="p">(</span><span class="n">out1</span> <span class="o">+</span> <span class="n">attn2</span><span class="p">)</span>
        
        <span class="c1"># ffn_output, out3.shape: (batch_size, target_seq_len, d_model)</span>
        <span class="n">ffn_output</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">ffn</span><span class="p">(</span><span class="n">out2</span><span class="p">)</span>
        <span class="n">ffn_output</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">dropout3</span><span class="p">(</span><span class="n">ffn_output</span><span class="p">,</span> <span class="n">training</span> <span class="o">=</span> <span class="n">training</span><span class="p">)</span>
        <span class="n">out3</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">layer_norm3</span><span class="p">(</span><span class="n">out2</span> <span class="o">+</span> <span class="n">ffn_output</span><span class="p">)</span>
        <span class="k">return</span> <span class="n">out3</span><span class="p">,</span> <span class="n">attn_weights1</span><span class="p">,</span> <span class="n">attn_weights2</span>
</pre></div>

    </div>
</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="prompt input_prompt">In&nbsp;[37]:</div>
<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="c1"># test </span>
<span class="n">sample_decoder_layer</span> <span class="o">=</span> <span class="n">DecoderLayer</span><span class="p">(</span><span class="mi">512</span><span class="p">,</span> <span class="mi">8</span><span class="p">,</span> <span class="mi">2048</span><span class="p">)</span>
<span class="n">sample_decoder_input</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">uniform</span><span class="p">((</span><span class="mi">64</span><span class="p">,</span> <span class="mi">60</span><span class="p">,</span> <span class="mi">512</span><span class="p">))</span>
<span class="n">sample_decoder_output</span><span class="p">,</span> <span class="n">sample_decoder_aw1</span><span class="p">,</span> <span class="n">sample_decoder_aw2</span> <span class="o">=</span> <span class="n">sample_decoder_layer</span><span class="p">(</span>
    <span class="n">sample_decoder_input</span><span class="p">,</span> <span class="n">sample_output</span><span class="p">,</span> <span class="kc">False</span><span class="p">,</span> <span class="kc">None</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>

<span class="nb">print</span><span class="p">(</span><span class="n">sample_decoder_output</span><span class="o">.</span><span class="n">shape</span><span class="p">)</span>
<span class="nb">print</span><span class="p">(</span><span class="n">sample_decoder_aw1</span><span class="o">.</span><span class="n">shape</span><span class="p">)</span>
<span class="nb">print</span><span class="p">(</span><span class="n">sample_decoder_aw2</span><span class="o">.</span><span class="n">shape</span><span class="p">)</span>
</pre></div>

    </div>
</div>
</div>

<div class="output_wrapper">
<div class="output">


<div class="output_area">

    <div class="prompt"></div>


<div class="output_subarea output_stream output_stdout output_text">
<pre>(64, 60, 512)
(64, 8, 60, 60)
(64, 8, 60, 50)
</pre>
</div>
</div>

</div>
</div>

</div>
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
</div><div class="inner_cell">
<div class="text_cell_render border-box-sizing rendered_html">
<h3 id="Encoder-Model">Encoder Model<a class="anchor-link" href="#Encoder-Model">&#182;</a></h3>
</div>
</div>
</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="prompt input_prompt">In&nbsp;[&nbsp;]:</div>
<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="k">class</span> <span class="nc">EncoderModel</span><span class="p">(</span><span class="n">keras</span><span class="o">.</span><span class="n">layers</span><span class="o">.</span><span class="n">Layer</span><span class="p">):</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">num_layers</span><span class="p">,</span> <span class="n">input_vocab_size</span><span class="p">,</span> <span class="n">max_length</span><span class="p">,</span>
                 <span class="n">d_model</span><span class="p">,</span> <span class="n">num_heads</span><span class="p">,</span> <span class="n">dff</span><span class="p">,</span> <span class="n">rate</span> <span class="o">=</span> <span class="mf">0.1</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">EncoderModel</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">d_model</span> <span class="o">=</span> <span class="n">d_model</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">num_layers</span> <span class="o">=</span> <span class="n">num_layers</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">max_length</span> <span class="o">=</span> <span class="n">max_length</span>
        
        <span class="bp">self</span><span class="o">.</span><span class="n">embedding</span> <span class="o">=</span> <span class="n">keras</span><span class="o">.</span><span class="n">layers</span><span class="o">.</span><span class="n">Embedding</span><span class="p">(</span><span class="n">input_vocab_size</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">d_model</span><span class="p">)</span>
        <span class="c1"># position_embedding.shape: (1, max_length, d_model)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">position_embedding</span> <span class="o">=</span> <span class="n">get_position_embedding</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">max_length</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">d_model</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">dropout</span> <span class="o">=</span> <span class="n">keras</span><span class="o">.</span><span class="n">layers</span><span class="o">.</span><span class="n">Dropout</span><span class="p">(</span><span class="n">rate</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">encoder_layers</span> <span class="o">=</span> <span class="p">[</span>
            <span class="n">EncoderLayer</span><span class="p">(</span><span class="n">d_model</span><span class="p">,</span> <span class="n">num_heads</span><span class="p">,</span> <span class="n">dff</span><span class="p">,</span> <span class="n">rate</span><span class="p">)</span> <span class="k">for</span> <span class="n">_</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">num_layers</span><span class="p">)]</span>
        
    <span class="k">def</span> <span class="nf">call</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">x</span><span class="p">,</span> <span class="n">training</span><span class="p">,</span> <span class="n">encoder_padding_mask</span><span class="p">):</span>
        <span class="c1"># x.shape: (batch_size, input_seq_len)</span>
        <span class="n">input_seq_len</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">shape</span><span class="p">(</span><span class="n">x</span><span class="p">)[</span><span class="mi">1</span><span class="p">]</span>
        <span class="n">tf</span><span class="o">.</span><span class="n">debugging</span><span class="o">.</span><span class="n">assert_less_equal</span><span class="p">(</span>
            <span class="n">input_seq_len</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_length</span><span class="p">,</span>
            <span class="n">message</span><span class="o">=</span><span class="s1">&#39;input_seq_len should be less or equal to self.max_length&#39;</span><span class="p">)</span>
        
        <span class="c1"># x.shape: (batch_size, input_seq_len, d_model)</span>
        <span class="n">x</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">embedding</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
        <span class="c1"># 做缩放，范围是0-d_model，目的是在与position_embedding做完加法后，x起的作用更大</span>
        <span class="n">x</span> <span class="o">*=</span> <span class="n">tf</span><span class="o">.</span><span class="n">math</span><span class="o">.</span><span class="n">sqrt</span><span class="p">(</span><span class="n">tf</span><span class="o">.</span><span class="n">cast</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">d_model</span><span class="p">,</span> <span class="n">tf</span><span class="o">.</span><span class="n">float32</span><span class="p">))</span>
        <span class="n">x</span> <span class="o">+=</span> <span class="bp">self</span><span class="o">.</span><span class="n">position_embedding</span><span class="p">[:,</span> <span class="p">:</span><span class="n">input_seq_len</span><span class="p">,</span> <span class="p">:]</span>
        <span class="n">x</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">dropout</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">training</span> <span class="o">=</span> <span class="n">training</span><span class="p">)</span>
        
        <span class="c1"># x.shape: (batch_size, input_seq_len, d_model)</span>
        <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">num_layers</span><span class="p">):</span>
            <span class="n">x</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">encoder_layers</span><span class="p">[</span><span class="n">i</span><span class="p">](</span><span class="n">x</span><span class="p">,</span> <span class="n">training</span><span class="p">,</span> <span class="n">encoder_padding_mask</span><span class="p">)</span>
            
        <span class="k">return</span> <span class="n">x</span>
</pre></div>

    </div>
</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="prompt input_prompt">In&nbsp;[39]:</div>
<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="c1"># test</span>
<span class="n">sample_encoder_model</span> <span class="o">=</span> <span class="n">EncoderModel</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">8500</span><span class="p">,</span> <span class="n">max_length</span><span class="p">,</span> <span class="mi">512</span><span class="p">,</span> <span class="mi">8</span><span class="p">,</span> <span class="mi">2048</span><span class="p">)</span>
<span class="n">sample_encoder_model_input</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">uniform</span><span class="p">((</span><span class="mi">64</span><span class="p">,</span> <span class="mi">37</span><span class="p">))</span>
<span class="n">sample_encoder_model_output</span> <span class="o">=</span> <span class="n">sample_encoder_model</span><span class="p">(</span>
    <span class="n">sample_encoder_model_input</span><span class="p">,</span> <span class="n">training</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">encoder_padding_mask</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span>

<span class="nb">print</span><span class="p">(</span><span class="n">sample_encoder_model_output</span><span class="o">.</span><span class="n">shape</span><span class="p">)</span>
</pre></div>

    </div>
</div>
</div>

<div class="output_wrapper">
<div class="output">


<div class="output_area">

    <div class="prompt"></div>


<div class="output_subarea output_stream output_stdout output_text">
<pre>(64, 37, 512)
</pre>
</div>
</div>

</div>
</div>

</div>
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
</div><div class="inner_cell">
<div class="text_cell_render border-box-sizing rendered_html">
<h3 id="Decoder-Model">Decoder Model<a class="anchor-link" href="#Decoder-Model">&#182;</a></h3>
</div>
</div>
</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="prompt input_prompt">In&nbsp;[&nbsp;]:</div>
<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="k">class</span> <span class="nc">DecoderModel</span><span class="p">(</span><span class="n">keras</span><span class="o">.</span><span class="n">layers</span><span class="o">.</span><span class="n">Layer</span><span class="p">):</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">num_layers</span><span class="p">,</span> <span class="n">target_vocab_size</span><span class="p">,</span> <span class="n">max_length</span><span class="p">,</span>
                 <span class="n">d_model</span><span class="p">,</span> <span class="n">num_heads</span><span class="p">,</span> <span class="n">dff</span><span class="p">,</span> <span class="n">rate</span> <span class="o">=</span> <span class="mf">0.1</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">DecoderModel</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">d_model</span> <span class="o">=</span> <span class="n">d_model</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">num_layers</span> <span class="o">=</span> <span class="n">num_layers</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">max_length</span> <span class="o">=</span> <span class="n">max_length</span>
        
        <span class="bp">self</span><span class="o">.</span><span class="n">embedding</span> <span class="o">=</span> <span class="n">keras</span><span class="o">.</span><span class="n">layers</span><span class="o">.</span><span class="n">Embedding</span><span class="p">(</span><span class="n">target_vocab_size</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">d_model</span><span class="p">)</span>
        <span class="c1"># position_embedding.shape: (1, max_length, d_model)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">position_embedding</span> <span class="o">=</span> <span class="n">get_position_embedding</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">max_length</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">d_model</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">dropout</span> <span class="o">=</span> <span class="n">keras</span><span class="o">.</span><span class="n">layers</span><span class="o">.</span><span class="n">Dropout</span><span class="p">(</span><span class="n">rate</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">decoder_layers</span> <span class="o">=</span> <span class="p">[</span>
            <span class="n">DecoderLayer</span><span class="p">(</span><span class="n">d_model</span><span class="p">,</span> <span class="n">num_heads</span><span class="p">,</span> <span class="n">dff</span><span class="p">,</span> <span class="n">rate</span><span class="p">)</span> <span class="k">for</span> <span class="n">_</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">num_layers</span><span class="p">)]</span>
        
    <span class="k">def</span> <span class="nf">call</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">x</span><span class="p">,</span> <span class="n">encoding_outputs</span><span class="p">,</span> <span class="n">training</span><span class="p">,</span> <span class="n">decoder_mask</span><span class="p">,</span> <span class="n">encoder_decoder_padding_mask</span><span class="p">):</span>
        <span class="c1"># x.shape: (batch_size, output_seq_len)</span>
        <span class="n">output_seq_len</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">shape</span><span class="p">(</span><span class="n">x</span><span class="p">)[</span><span class="mi">1</span><span class="p">]</span>
        <span class="n">tf</span><span class="o">.</span><span class="n">debugging</span><span class="o">.</span><span class="n">assert_less_equal</span><span class="p">(</span>
            <span class="n">output_seq_len</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_length</span><span class="p">,</span>
            <span class="n">message</span><span class="o">=</span><span class="s1">&#39;output_seq_len should be less or equal to self.max_length&#39;</span><span class="p">)</span>
        
        <span class="n">attention_weights</span> <span class="o">=</span> <span class="p">{}</span>
        
        <span class="c1"># x.shape: (batch_size, output_seq_len, d_model)</span>
        <span class="n">x</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">embedding</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
        <span class="c1"># 做缩放，范围是0-d_model，目的是在与position_embedding做完加法后，x起的作用更大</span>
        <span class="n">x</span> <span class="o">*=</span> <span class="n">tf</span><span class="o">.</span><span class="n">math</span><span class="o">.</span><span class="n">sqrt</span><span class="p">(</span><span class="n">tf</span><span class="o">.</span><span class="n">cast</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">d_model</span><span class="p">,</span> <span class="n">tf</span><span class="o">.</span><span class="n">float32</span><span class="p">))</span>
        <span class="n">x</span> <span class="o">+=</span> <span class="bp">self</span><span class="o">.</span><span class="n">position_embedding</span><span class="p">[:,</span> <span class="p">:</span><span class="n">output_seq_len</span><span class="p">,</span> <span class="p">:]</span>
        <span class="n">x</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">dropout</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">training</span> <span class="o">=</span> <span class="n">training</span><span class="p">)</span>
        
        <span class="c1"># x.shape: (batch_size, output_seq_len, d_model)</span>
        <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">num_layers</span><span class="p">):</span>
            <span class="n">x</span><span class="p">,</span> <span class="n">att1</span><span class="p">,</span> <span class="n">att2</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">decoder_layers</span><span class="p">[</span><span class="n">i</span><span class="p">](</span>
                <span class="n">x</span><span class="p">,</span> <span class="n">encoding_outputs</span><span class="p">,</span> <span class="n">training</span><span class="p">,</span> <span class="n">decoder_mask</span><span class="p">,</span> <span class="n">encoder_decoder_padding_mask</span><span class="p">)</span>
            <span class="n">attention_weights</span><span class="p">[</span><span class="s1">&#39;decoder_layer</span><span class="si">{}</span><span class="s1">_att1&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">i</span><span class="o">+</span><span class="mi">1</span><span class="p">)]</span> <span class="o">=</span> <span class="n">att1</span>
            <span class="n">attention_weights</span><span class="p">[</span><span class="s1">&#39;decoder_layer</span><span class="si">{}</span><span class="s1">_att2&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">i</span><span class="o">+</span><span class="mi">1</span><span class="p">)]</span> <span class="o">=</span> <span class="n">att2</span>
        
        <span class="k">return</span> <span class="n">x</span><span class="p">,</span> <span class="n">attention_weights</span>
</pre></div>

    </div>
</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="prompt input_prompt">In&nbsp;[41]:</div>
<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="c1"># test</span>
<span class="n">sample_decoder_model</span> <span class="o">=</span> <span class="n">DecoderModel</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">8000</span><span class="p">,</span> <span class="n">max_length</span><span class="p">,</span> <span class="mi">512</span><span class="p">,</span> <span class="mi">8</span><span class="p">,</span> <span class="mi">2048</span><span class="p">)</span>
<span class="n">sample_decoder_model_input</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">uniform</span><span class="p">((</span><span class="mi">64</span><span class="p">,</span> <span class="mi">35</span><span class="p">))</span>
<span class="n">sample_decoder_model_output</span><span class="p">,</span> <span class="n">sample_decoder_model_att</span> <span class="o">=</span> <span class="n">sample_decoder_model</span><span class="p">(</span>
    <span class="n">sample_decoder_model_input</span><span class="p">,</span> <span class="n">sample_encoder_model_output</span><span class="p">,</span>
    <span class="n">training</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span> <span class="n">decoder_mask</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="n">encoder_decoder_padding_mask</span> <span class="o">=</span> <span class="kc">None</span><span class="p">)</span>

<span class="nb">print</span><span class="p">(</span><span class="n">sample_decoder_model_output</span><span class="o">.</span><span class="n">shape</span><span class="p">)</span>
<span class="k">for</span> <span class="n">key</span> <span class="ow">in</span> <span class="n">sample_decoder_model_att</span><span class="p">:</span>
    <span class="nb">print</span><span class="p">(</span><span class="n">sample_decoder_model_att</span><span class="p">[</span><span class="n">key</span><span class="p">]</span><span class="o">.</span><span class="n">shape</span><span class="p">)</span>
</pre></div>

    </div>
</div>
</div>

<div class="output_wrapper">
<div class="output">


<div class="output_area">

    <div class="prompt"></div>


<div class="output_subarea output_stream output_stdout output_text">
<pre>(64, 35, 512)
(64, 8, 35, 35)
(64, 8, 35, 37)
(64, 8, 35, 35)
(64, 8, 35, 37)
</pre>
</div>
</div>

</div>
</div>

</div>
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
</div><div class="inner_cell">
<div class="text_cell_render border-box-sizing rendered_html">
<h3 id="Transformer">Transformer<a class="anchor-link" href="#Transformer">&#182;</a></h3>
</div>
</div>
</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="prompt input_prompt">In&nbsp;[&nbsp;]:</div>
<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="k">class</span> <span class="nc">Transformer</span><span class="p">(</span><span class="n">keras</span><span class="o">.</span><span class="n">Model</span><span class="p">):</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">num_layers</span><span class="p">,</span> <span class="n">input_vocab_size</span><span class="p">,</span> <span class="n">target_vocab_size</span><span class="p">,</span> <span class="n">max_length</span><span class="p">,</span>
                 <span class="n">d_model</span><span class="p">,</span> <span class="n">num_heads</span><span class="p">,</span> <span class="n">dff</span><span class="p">,</span> <span class="n">rate</span> <span class="o">=</span> <span class="mf">0.1</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">Transformer</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">encoder_model</span> <span class="o">=</span> <span class="n">EncoderModel</span><span class="p">(</span><span class="n">num_layers</span><span class="p">,</span> <span class="n">input_vocab_size</span><span class="p">,</span> <span class="n">max_length</span><span class="p">,</span>
                                          <span class="n">d_model</span><span class="p">,</span> <span class="n">num_heads</span><span class="p">,</span> <span class="n">dff</span><span class="p">,</span> <span class="n">rate</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">decoder_model</span> <span class="o">=</span> <span class="n">DecoderModel</span><span class="p">(</span><span class="n">num_layers</span><span class="p">,</span> <span class="n">target_vocab_size</span><span class="p">,</span> <span class="n">max_length</span><span class="p">,</span>
                                          <span class="n">d_model</span><span class="p">,</span> <span class="n">num_heads</span><span class="p">,</span> <span class="n">dff</span><span class="p">,</span> <span class="n">rate</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">final_layer</span> <span class="o">=</span> <span class="n">keras</span><span class="o">.</span><span class="n">layers</span><span class="o">.</span><span class="n">Dense</span><span class="p">(</span><span class="n">target_vocab_size</span><span class="p">)</span>
        
    <span class="k">def</span> <span class="nf">call</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">inp</span><span class="p">,</span> <span class="n">tar</span><span class="p">,</span> <span class="n">training</span><span class="p">,</span> <span class="n">encoder_padding_mask</span><span class="p">,</span>
             <span class="n">decoder_mask</span><span class="p">,</span> <span class="n">encoder_decoder_padding_mask</span><span class="p">):</span>
        <span class="c1"># encoding_outputs.shape: (batch_size, input_seq_len, d_model)</span>
        <span class="n">encoding_outputs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">encoder_model</span><span class="p">(</span><span class="n">inp</span><span class="p">,</span> <span class="n">training</span><span class="p">,</span> <span class="n">encoder_padding_mask</span><span class="p">)</span>
        
        <span class="c1"># decoding_outputs.shape: (batch_size, output_seq_len, d_model)</span>
        <span class="n">decoding_outputs</span><span class="p">,</span> <span class="n">attention_weights</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">decoder_model</span><span class="p">(</span>
            <span class="n">tar</span><span class="p">,</span> <span class="n">encoding_outputs</span><span class="p">,</span> <span class="n">training</span><span class="p">,</span> <span class="n">decoder_mask</span><span class="p">,</span> <span class="n">encoder_decoder_padding_mask</span><span class="p">)</span>
        
        <span class="c1"># decoding_outputs.shape: (batch_size, output_seq_len, target_vocab_size)</span>
        <span class="n">predictions</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">final_layer</span><span class="p">(</span><span class="n">decoding_outputs</span><span class="p">)</span>
        
        <span class="k">return</span> <span class="n">predictions</span><span class="p">,</span> <span class="n">attention_weights</span>
</pre></div>

    </div>
</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="prompt input_prompt">In&nbsp;[43]:</div>
<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="c1"># test</span>
<span class="n">sample_transformer</span> <span class="o">=</span> <span class="n">Transformer</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">8500</span><span class="p">,</span> <span class="mi">8000</span><span class="p">,</span> <span class="n">max_length</span><span class="p">,</span> <span class="mi">512</span><span class="p">,</span> <span class="mi">8</span><span class="p">,</span> <span class="mi">2048</span><span class="p">,</span> <span class="n">rate</span><span class="o">=</span><span class="mf">0.1</span><span class="p">)</span>

<span class="n">temp_input</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">uniform</span><span class="p">((</span><span class="mi">64</span><span class="p">,</span> <span class="mi">26</span><span class="p">))</span>
<span class="n">temp_target</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">uniform</span><span class="p">((</span><span class="mi">64</span><span class="p">,</span> <span class="mi">31</span><span class="p">))</span>

<span class="n">predictions</span><span class="p">,</span> <span class="n">attention_weights</span> <span class="o">=</span> <span class="n">sample_transformer</span><span class="p">(</span>
    <span class="n">temp_input</span><span class="p">,</span> <span class="n">temp_target</span><span class="p">,</span> <span class="n">training</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span> <span class="n">encoder_padding_mask</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
    <span class="n">decoder_mask</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="n">encoder_decoder_padding_mask</span> <span class="o">=</span> <span class="kc">None</span><span class="p">)</span>

<span class="nb">print</span><span class="p">(</span><span class="n">predictions</span><span class="o">.</span><span class="n">shape</span><span class="p">)</span>
<span class="k">for</span> <span class="n">key</span> <span class="ow">in</span> <span class="n">attention_weights</span><span class="p">:</span>
    <span class="nb">print</span><span class="p">(</span><span class="n">key</span><span class="p">,</span> <span class="n">attention_weights</span><span class="p">[</span><span class="n">key</span><span class="p">]</span><span class="o">.</span><span class="n">shape</span><span class="p">)</span>
</pre></div>

    </div>
</div>
</div>

<div class="output_wrapper">
<div class="output">


<div class="output_area">

    <div class="prompt"></div>


<div class="output_subarea output_stream output_stdout output_text">
<pre>(64, 31, 8000)
decoder_layer1_att1 (64, 8, 31, 31)
decoder_layer1_att2 (64, 8, 31, 26)
decoder_layer2_att1 (64, 8, 31, 31)
decoder_layer2_att2 (64, 8, 31, 26)
</pre>
</div>
</div>

</div>
</div>

</div>
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
</div><div class="inner_cell">
<div class="text_cell_render border-box-sizing rendered_html">
<h2 id="Train">Train<a class="anchor-link" href="#Train">&#182;</a></h2><h3 id="initializes-model">initializes model<a class="anchor-link" href="#initializes-model">&#182;</a></h3>
</div>
</div>
</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="prompt input_prompt">In&nbsp;[&nbsp;]:</div>
<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="c1"># 定义超参数</span>
<span class="n">num_layers</span> <span class="o">=</span> <span class="mi">4</span>
<span class="n">d_model</span> <span class="o">=</span> <span class="mi">128</span>
<span class="n">dff</span> <span class="o">=</span> <span class="mi">512</span>
<span class="n">num_heads</span> <span class="o">=</span> <span class="mi">8</span>
<span class="n">dropout_rate</span> <span class="o">=</span> <span class="mf">0.1</span>

<span class="n">input_vocab_size</span> <span class="o">=</span> <span class="n">pt_tokenizer</span><span class="o">.</span><span class="n">vocab_size</span> <span class="o">+</span> <span class="mi">2</span>
<span class="n">target_vocab_size</span> <span class="o">=</span> <span class="n">en_tokenizer</span><span class="o">.</span><span class="n">vocab_size</span> <span class="o">+</span> <span class="mi">2</span>
</pre></div>

    </div>
</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="prompt input_prompt">In&nbsp;[&nbsp;]:</div>
<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="c1"># model</span>
<span class="n">transformer</span> <span class="o">=</span> <span class="n">Transformer</span><span class="p">(</span><span class="n">num_layers</span><span class="p">,</span> <span class="n">input_vocab_size</span><span class="p">,</span> <span class="n">target_vocab_size</span><span class="p">,</span>
                          <span class="n">max_length</span><span class="p">,</span> <span class="n">d_model</span><span class="p">,</span> <span class="n">num_heads</span><span class="p">,</span> <span class="n">dff</span><span class="p">,</span> <span class="n">dropout_rate</span><span class="p">)</span>
</pre></div>

    </div>
</div>
</div>

</div>
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
</div><div class="inner_cell">
<div class="text_cell_render border-box-sizing rendered_html">
<h3 id="&#33258;&#23450;&#20041;&#23398;&#20064;&#29575;">&#33258;&#23450;&#20041;&#23398;&#20064;&#29575;<a class="anchor-link" href="#&#33258;&#23450;&#20041;&#23398;&#20064;&#29575;">&#182;</a></h3><p>learning_rate 先增后减</p>
$$
lrate = (d\_model^{-0.5}) * min(step\_num^{-0.5}, step\_num * warm\_up\_steps^{-1.5})
$$<p></p>

</div>
</div>
</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="prompt input_prompt">In&nbsp;[&nbsp;]:</div>
<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="k">class</span> <span class="nc">CustomizedSchedule</span><span class="p">(</span><span class="n">keras</span><span class="o">.</span><span class="n">optimizers</span><span class="o">.</span><span class="n">schedules</span><span class="o">.</span><span class="n">LearningRateSchedule</span><span class="p">):</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">d_model</span><span class="p">,</span> <span class="n">warmup_steps</span> <span class="o">=</span> <span class="mi">4000</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">CustomizedSchedule</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">d_model</span> <span class="o">=</span> <span class="n">d_model</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">warmup_steps</span> <span class="o">=</span> <span class="n">warmup_steps</span>
        
    <span class="k">def</span> <span class="fm">__call__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">step</span><span class="p">):</span>
        <span class="n">arg1</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">math</span><span class="o">.</span><span class="n">rsqrt</span><span class="p">(</span><span class="n">step</span><span class="p">)</span>
        <span class="n">arg2</span> <span class="o">=</span> <span class="n">step</span> <span class="o">*</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">warmup_steps</span> <span class="o">**</span> <span class="p">(</span><span class="o">-</span><span class="mf">1.5</span><span class="p">))</span>
        <span class="n">arg3</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">math</span><span class="o">.</span><span class="n">rsqrt</span><span class="p">(</span><span class="n">tf</span><span class="o">.</span><span class="n">cast</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">d_model</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="n">tf</span><span class="o">.</span><span class="n">float32</span><span class="p">))</span>
        <span class="k">return</span> <span class="n">arg3</span> <span class="o">*</span> <span class="n">tf</span><span class="o">.</span><span class="n">math</span><span class="o">.</span><span class="n">minimum</span><span class="p">(</span><span class="n">arg1</span><span class="p">,</span> <span class="n">arg2</span><span class="p">)</span>
    
<span class="n">learning_rate</span> <span class="o">=</span> <span class="n">CustomizedSchedule</span><span class="p">(</span><span class="n">d_model</span><span class="p">)</span>
<span class="n">optimizer</span> <span class="o">=</span> <span class="n">keras</span><span class="o">.</span><span class="n">optimizers</span><span class="o">.</span><span class="n">Adam</span><span class="p">(</span><span class="n">learning_rate</span><span class="p">,</span> <span class="n">beta_1</span><span class="o">=</span><span class="mf">0.9</span><span class="p">,</span> <span class="n">beta_2</span><span class="o">=</span><span class="mf">0.98</span><span class="p">,</span> <span class="n">epsilon</span><span class="o">=</span><span class="mf">1e-9</span><span class="p">)</span>
</pre></div>

    </div>
</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="prompt input_prompt">In&nbsp;[47]:</div>
<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="c1"># 画出learning_rate变化图示</span>
<span class="n">temp_learning_rate</span> <span class="o">=</span> <span class="n">CustomizedSchedule</span><span class="p">(</span><span class="n">d_model</span><span class="p">)</span>

<span class="n">plt</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">temp_learning_rate</span><span class="p">(</span><span class="n">tf</span><span class="o">.</span><span class="n">range</span><span class="p">(</span><span class="mi">40000</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="n">tf</span><span class="o">.</span><span class="n">float32</span><span class="p">)))</span>
<span class="n">plt</span><span class="o">.</span><span class="n">ylabel</span><span class="p">(</span><span class="s1">&#39;Learning rate&#39;</span><span class="p">)</span>
<span class="n">plt</span><span class="o">.</span><span class="n">xlabel</span><span class="p">(</span><span class="s1">&#39;Train step&#39;</span><span class="p">)</span>
</pre></div>

    </div>
</div>
</div>

<div class="output_wrapper">
<div class="output">


<div class="output_area">

    <div class="prompt output_prompt">Out[47]:</div>




<div class="output_text output_subarea output_execute_result">
<pre>Text(0.5, 0, &#39;Train step&#39;)</pre>
</div>

</div>

<div class="output_area">

    <div class="prompt"></div>




<div class="output_png output_subarea ">
<img src="
AAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0
dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3deXxU9bn48c+TnYQkZGMNkBDCElxQ
I651wwVtr9z2agv13kvVSm/Vrrd1+fX3s73+6r3a9la7aPtz11YFSrWlve5LXaoiAQUBRTIDQtgy
CRBJgIQkz++P8w0McZJMkpnMJPO8X6+8cuZ7zvmeZyaQJ+d8v+c5oqoYY4wxkZAU6wCMMcYMHZZU
jDHGRIwlFWOMMRFjScUYY0zEWFIxxhgTMSmxDiCWCgsLtaSkJNZhGGPMoLJy5co6VS0KtS6hk0pJ
SQlVVVWxDsMYYwYVEfm4q3V2+csYY0zEWFIxxhgTMZZUjDHGRIwlFWOMMRFjScUYY0zERDWpiMgc
EdkgItUiclOI9ekistitXy4iJUHrbnbtG0TkoqD2B0WkVkTWdnHMfxcRFZHCaLwnY4wxXYtaUhGR
ZOBu4GKgApgvIhWdNrsa2KOqk4E7gTvcvhXAPGAGMAe4x/UH8LBrC3XM8cCFwJaIvhljjDFhieaZ
yiygWlX9qtoCLALmdtpmLvCIW14KzBYRce2LVLVZVTcB1a4/VPU1YHcXx7wTuAEYkvX8VZUlK7bS
2Nwa61CMMSakaCaVccDWoNc1ri3kNqraCjQABWHuexQRmQtsU9XVPWy3UESqRKQqEAiE8z7ixntb
93LDH9dw49I1sQ7FGGNCGhID9SKSCfwv4JaetlXVe1W1UlUri4pCVhmIW1t27wfghQ92xTgSY4wJ
LZpJZRswPuh1sWsLuY2IpAC5QH2Y+wYrA0qB1SKy2W2/SkRG9yP+uOMLNAHQ0trOVpdgjDEmnkQz
qawAykWkVETS8Abel3XaZhmwwC1fBrys3vONlwHz3OywUqAceKerA6nq+6o6UlVLVLUE73LZiaq6
M7JvKbZ8gUZEvOVn1u6IbTDGGBNC1JKKGyO5HngO+ABYoqrrRORWEbnUbfYAUCAi1cB3gZvcvuuA
JcB64FngOlVtAxCRJ4C3gKkiUiMiV0frPcQbf6CJs6cUMWNsDs+sHVL50hgzRES1SrGqPg083ant
lqDlg8DlXex7G3BbiPb5YRy3pLexxrv2dmVTXSOnlxVwckk+P31uAzsaDjAmd1isQzPGmMOGxEB9
ItjecICDh9qZVJTFnGO8oaJn7WzFGBNnLKkMEn43SF9WNJyyouFMG53NX1Zvj3FUxhhzNEsqg4Qv
0AjApKIsAObOHMeqLXv5uL4plmEZY8xRLKkMEv5AE9kZKRQNTwdg7syxiMCf3rWzFWNM/LCkMkj4
Ao1MKhqOuDnFY0cM49TSAp56twZvFrYxxsSeJZVBwh9ooqww66i2z584js31+3l3694YRWWMMUez
pDIINDa3svOTg5SNHH5U+8XHjCY9JYk/vdtdsQFjjBk4llQGgU1u5tekTmcq2RmpXFAxir+s3k5z
a1ssQjPGmKNYUhkE/HXezK/OZyoAl1eOZ8/+Qzy/zopMGmNiz5LKIOCrbSRJYGJB5qfWfWZyIcV5
w3h8uT2XzBgTe5ZUBgFfXRPFeZmkpyR/al1SkjB/1gTe8tfjd/eyGGNMrFhSGQR8tY2UFWV1uf7y
ymJSkoRFK7Z2uY0xxgwESypxrr1d2VzfxKSiT4+ndBiZncH500exdGWNDdgbY2LKkkqc6ygkWdZN
UgH48ikT2N3UYkUmjTExZUklznU87XFSN5e/AM6cXEhpYRYP/n2z3WFvjIkZSypxrmPwvaczlaQk
4cozSli9dS+rtuwZiNCMMeZTLKnEOV+gkeyMFAqHp/W47WUnFZM7LJX7X980AJEZY8ynWVKJc/5A
01GFJLuTmZbC/FkTeG7dTrbu3j8A0RljzNEsqcQ5f6Cp2+nEnS04fSJJIjz85uboBWWMMV2IalIR
kTkiskFEqkXkphDr00VksVu/XERKgtbd7No3iMhFQe0PikitiKzt1NdPReRDEVkjIk+JyIhovreB
cLiQZA/jKcHG5A7jkmPHsHjFVhr2H4pidMYY82lRSyoikgzcDVwMVADzRaSi02ZXA3tUdTJwJ3CH
27cCmAfMAOYA97j+AB52bZ29AByjqscBHwE3R/QNxcCmw48QDv9MBeDr55TR2NzKQ2/a2IoxZmBF
80xlFlCtqn5VbQEWAXM7bTMXeMQtLwVmizd4MBdYpKrNqroJqHb9oaqvAbs7H0xVn1fVVvfybaA4
0m9ooB15hHD4ZyoA08fkcP70UTz0983sO2hnK8aYgRPNpDIOCK4bUuPaQm7jEkIDUBDmvt25Cngm
1AoRWSgiVSJSFQgEetHlwPMHui4k2ZNvzp5Mw4FD/O7tj6MQmTHGhDbkBupF5AdAK/BYqPWqeq+q
VqpqZVFR0cAG10u+QBPj80MXkuzJccUjOHtKEfe/von9La0972CMMREQzaSyDRgf9LrYtYXcRkRS
gFygPsx9P0VEvgJ8DrhCh8Bt5b5A46cezNUb3zhvMrubWnjsbSuLb4wZGNFMKiuAchEpFZE0vIH3
ZZ22WQYscMuXAS+7ZLAMmOdmh5UC5cA73R1MROYANwCXquqgv0mjvV3ZVNfUq5lfnVWW5HPm5EJ+
86rPxlaMMQMiaknFjZFcDzwHfAAsUdV1InKriFzqNnsAKBCRauC7wE1u33XAEmA98Cxwnaq2AYjI
E8BbwFQRqRGRq11fvwaygRdE5D0R+W203ttA2Lb3AM2t7b0epO/sxjnT2N3Uwn2v+SMUmTHGdC0l
mp2r6tPA053abglaPghc3sW+twG3hWif38X2k/sVbJzx1/VtOnFnxxbn8tnjxnD/G5v4l9NKKMpO
j0R4xhgT0pAbqB8qfLV9m04cyvcunEpLazu/enljv/syxpjuWFKJU/668AtJ9qS0MIsvnTyex5dv
YbM7AzLGmGiwpBKnvJpf4RWSDMe3ZpeTnpLEj//ng4j0Z4wxoVhSiVO+QGOPD+bqjZE5GXxjdjkv
frCLv22ojVi/xhgTzJJKHGpsbmXXJ839mk4cypVnlFBamMWtf1lPS2t7RPs2xhiwpBKXjjztMXJn
KgDpKcnc8g8V+OuaeNiKTRpjosCSShzyH34ufWTPVADOnTqS2dNG8osXN7Kz4WDE+zfGJDZLKnHI
149CkuG45R8qaFPl//x5LUOgmo0xJo5YUolD/n4UkgzHxIIsvnP+FF5Yv4tn1u6MyjGMMYnJkkoc
8gUaIz5I39nVZ5ZyzLgcbvnzOntCpDEmYiypxJmOQpL9qU4cjpTkJO74p+PYs7+F255eH9VjGWMS
hyWVONNRSLJsZHTPVABmjM1l4VmTWFJVwyt274oxJgIsqcSZw48QjvKZSodvzS5n6qhsbli6hvrG
5gE5pjFm6LKkEmeiOZ04lIzUZO6aN5OG/Ye4+cn3bTaYMaZfLKnEGX9dIzkRKiQZruljcrhhzlSe
X7+LJVVbB+y4xpihx5JKnPHVNjEpgoUkw3XVGaWcXlbAf/xl/eE7+o0xprcsqcQZf130pxOHkpQk
/PcXjyc9JYlrH1vFgZa2AY/BGDP4WVKJI/sOHmLXJ80RrU7cG2Nyh3Hnl2ayYdc+/vef7G57Y0zv
WVKJI5si9Ajh/jhn6ki+cV45f1xVw+IVNr5ijOmdqCYVEZkjIhtEpFpEbgqxPl1EFrv1y0WkJGjd
za59g4hcFNT+oIjUisjaTn3li8gLIrLRfc+L5nuLBt/h6sQDf/kr2Ldml/OZ8kJuWbaOtdsaYhqL
MWZwiVpSEZFk4G7gYqACmC8iFZ02uxrYo6qTgTuBO9y+FcA8YAYwB7jH9QfwsGvr7CbgJVUtB15y
rwcVf6CJJIEJUSokGa7kJOGuL82kMCuNax6tonafVTM2xoQnmmcqs4BqVfWraguwCJjbaZu5wCNu
eSkwW7xpT3OBRararKqbgGrXH6r6GrA7xPGC+3oE+MdIvpmB4A80MSGKhSR7o2B4OvctqGTv/kMs
fHQlBw/ZwL0xpmfRTCrjgOCL8jWuLeQ2qtoKNAAFYe7b2ShV3eGWdwKjQm0kIgtFpEpEqgKBQDjv
Y8B4jxCO7aWvYDPG5nLXvJm8t3UvNyxdYwP3xpgeDcmBevV++4X8Daiq96pqpapWFhUVDXBkXWtz
hSRjOUgfykUzRnPDnKksW72dX71cHetwjDFxLppJZRswPuh1sWsLuY2IpAC5QH2Y+3a2S0TGuL7G
AIOqQuJ2V0gyns5UOnz97DK+cOI4fv7CRyyxGWHGmG5EM6msAMpFpFRE0vAG3pd12mYZsMAtXwa8
7M4ylgHz3OywUqAceKeH4wX3tQD4cwTew4AZ6EKSvSEi3P6F4zhrShE3PbmGF9bvinVIxpg4FbWk
4sZIrgeeAz4AlqjqOhG5VUQudZs9ABSISDXwXdyMLVVdBywB1gPPAtepahuAiDwBvAVMFZEaEbna
9XU7cIGIbATOd68HjY5CkgNR8r4v0lKS+M0VJ3Js8Qiuf3wV72wKNVfCGJPoJJEHXysrK7WqqirW
YQDwg6fe5y+rt7P6hxcOeN2v3tjd1MJlv32TwL5mFi88jYqxObEOyRgzwERkpapWhlo3JAfqByN/
oImykQNfSLK38rPSePSqWQxPT+GK+9/mgx2fxDokY0wcsaQSJ3yBRiYVxuelr86K8zJ54ppTSU9J
5or7l7Nh575Yh2SMiROWVOLAvoOHqN0Xu0KSfVFSmMUTC08lNVn48n1vs3GXJRZjjCWVuHB4kD4O
pxN3p7Qwi8evOZXkJGH+fXYpzBhjSSUu+Os6CkkOnjOVDmVFw3li4amkJCXxpf/3Fis/tllhxiQy
SypxwB9oIjlJYl5Isq/Kioaz9OunUTA8nSvuX87fNgyq+06NMRHUY1IRkUwR+T8icp97XS4in4t+
aInDF2hkfN6wuCgk2VfFeZks+dppTCoczjWPVvGX1dtjHZIxJgbCOVN5CGgGTnOvtwE/jlpECcgf
aBp04ymhFGWns+hrp3LC+Dy+uehd7n3NZ0UojUkw4SSVMlX9CXAIQFX3A/F9M8Ug0tau+OuaBtXM
r+7kZKTy6NWzuOSYMfzn0x/yv55ay6G29liHZYwZIClhbNMiIsNwVX9FpAzvzMVEwPa9B2iJ00KS
fZWRmsyv5p/AxIJM7vmbj5o9+7n7ihPJyUiNdWjGmCgL50zlR3j1t8aLyGN4T1W8MZpBJZJ4eYRw
pCUlCTfMmcZPLjuOt3z1/NM9b7K5rinWYRljoqzHpKKqzwNfAL4CPAFUquorUY4rYfjcPSpD5fJX
Z1+sHM+jV88i0NjMP/z6DV76wCocGzOUhTP76yVVrVfV/1HVv6pqnYi8NBDBJQJ/oJHcYakUZKXF
OpSoOb2skL9cfyYTCzK5+pEqfv78BtrabQDfmKGoy6QiIhkikg8UikieiOS7rxJ6frSvCZP3COGs
uC8k2V/j8zNZ+m+nc/lJxfzy5WquengFe5paYh2WMSbCujtT+RqwEpjmvnd8/Rn4dfRDSwz+QNOg
KSTZXxmpyfzksuO47fPH8Kavjkt++Tpv++tjHZYxJoK6TCqq+gtVLQW+p6qTVLXUfR2vqpZUIqCj
kGTZyKE5nhKKiHDFKRN58utnkJGazPz73ubnz2+g1aYdGzMk9DilWFV/JSLHABVARlD7o9EMLBF0
FJJMlDOVYMcW5/LXb5zJD5et45cvV/N3Xz13fWkm4/MHZ6kaY4wnnIH6HwK/cl/nAj8BLu12JxOW
jkKSkxPoTCVYVnoKP7v8eH45/wQ+2rmPS37xOkuqttpd+MYMYuHcp3IZMBvYqapXAscDuVGNKkH4
al0hyfzETCodLj1+LE9/6zNMH5vDDUvX8JWHVrB974FYh2WM6YNwksoBVW0HWkUkB6gFxkc3rMTg
r2tkQn4maSlWLHp8fiaLrjmV/7h0Bu9s2s1Fd77G4hVb7KzFmEEmnN9mVSIyArgPb/bXKuCtcDoX
kTkiskFEqkXkphDr00VksVu/3E1X7lh3s2vfICIX9dSniMwWkVUi8p6IvCEik8OJMZZ8tU1MKkzs
s5RgSUnCgtNLeO7bZ1ExNocb//g+//rgO3YnvjGDSLdJRbybJ/5LVfeq6m+BC4AF7jJYt0QkGbgb
uBhvkH++iFR02uxqYI+qTgbuBO5w+1YA84AZwBzgHhFJ7qHP3wBXqOpM4HHgf/f47mOorV3ZVD90
CklG0oSCTJ645lRunTuDd7fs5cK7XuMXL26kubUt1qEZY3rQbVJR79rD00GvN6vqmjD7ngVUq6pf
VVuARcDcTtvMBR5xy0uB2S6RzQUWqWqzqm4Cql1/3fWpQI5bzgXi+oEeHYUkh1rNr0hJShL+9bQS
Xvr3s7mwYhR3vvgRc+56nTc21sU6NGNMN8K5/LVKRE7uQ9/jgK1Br2v49J34h7dR1VagASjoZt/u
+vwq8LSI1AD/AtweKigRWSgiVSJSFQgE+vC2IqPaFZIcStWJo2FUTga//vKJPHrVLFSVf35gOdc/
voptNpBvTFwKJ6mcArwlIj4RWSMi74tIuGcrA+k7wCWqWoz3YLGfh9pIVe9V1UpVrSwqKhrQAIN1
3KMyGJ9LHwtnTSni2W+fxbfPL+eF9bs472d/46fPfUhjc2usQzPGBAnneSoX9bxJSNs4epZYsWsL
tU2NiKTgXbaq72HfT7WLSBFwvKoud+2L8cr1xy2fKySZP4QLSUZaRmoy3z5/CpdXjuenz37I3a/4
WFJVw/cunMJlJ40nOWlo108zZjAIp/T9x6G+wuh7BVAuIqUikoY38L6s0zbLgAVu+TLgZTeOswyY
52aHlQLlwDvd9LkHyBWRKa6vC4APwogxZvwJUkgyGsaNGMZd807gqWtPZ0J+Jjf+8X0++8vX+duG
WpuCbEyMhXOm0ieq2ioi1wPPAcnAg6q6TkRuBapUdRnwAPA7EakGduMlCdx2S4D1QCtwnaq2AYTq
07VfA/xRRNrxksxV0XpvkeALNHH2lNhdfhsKTpiQx9J/O42n39/J7c9+wFceWsHJJXl878KpnDKp
INbhGZOQJJH/squsrNSqqqoBP+6+g4c49kfPc8OcqVx7TtzfTjMotLS2s7hqK79+eSO7PmnmM+WF
fO/CqRw/fkSsQzNmyBGRlapaGWqd3codA0cG6W3mV6SkpSTxL6dO5NXvn8sPLpnOuu2fMPfuv3PN
o1Wsqdkb6/CMSRg9Xv4SkX1494AEawCqgH9XVX80AhvKjjyX3mZ+RVpGajLXnDWJ+adM4ME3NnHf
635eWL+Lz5QXct25kzmlNN/GsYyJonDGVO7Cux/kcUDwxj3K8Mq1PAicE63ghip/wApJRtvw9BS+
ObucK88o4fdvb+GBN/zMu/dtTpqYx3XnlnHu1JGWXIyJgnAuf12qqv9PVfep6ieqei9wkaouBvKi
HN+Q5AtYIcmBkp2RytfPKeONG8/j1rkz2NlwkKseruLiX7zOU+/W0NJqDwczJpLC+a22X0S+KCJJ
7uuLwEG3LnFH+fvBe4SwnaUMpIzUZP71tBL+9v1z+Nnlx3OorZ3vLF7NGXe8zK9e2kh9Y3OsQzRm
SAgnqVyBV/akFtjllv9ZRIYB10cxtiGpo5Bk2UgbpI+F1OQkLjupmBe+czYPX3ky08fk8N8vfMRp
t7/MjUvX8OHOT2IdojGDWjiPE/YD/9DF6jciG87Qt22PV0jSzlRiKylJOGfqSM6ZOpKNu/bx0Jub
eXJVDYurtnJ6WQH/fOpELqgYRWqyXaI0pjfCmf1VBFwDlARvr6pxfXNhvPK5RwjbmUr8KB+VzX9+
/li+f+FUnlixhd+/9THXPraKwuHpfLGymPmzJjA+PzPWYRozKIQz++vPwOvAi4A90KKffLWuOrGd
qcSdvKw0rj1nMl87q4xXP6rl8eVb+O2rPn7zqo/PlBfx5VkTmD19pJ29GNONcJJKpqreGPVIEoS/
rokRmVZIMp4lJwnnTRvFedNGsX3vARav2MriFVv5t9+vpCg7nX+cOZYvnFjM9DE5PXdmTIIJJ6n8
VUQuUdWne97U9MRX28ikQiskOViMHTGM71wwhW+cN5lXNgT4Q9VWHn5zM/e9vomKMTl84cRxzJ05
jqLs9FiHakxc6LH2l7ujPgtoBg7h3QCpqjro/0yLRe2vk297kbOnFPGzy48f0OOayNnd1MJfVm/n
yVU1rK5pIDlJOHtKEV84cRznTx9FRmpyrEM0Jqq6q/0Vzuyv7MiHlJg+OXiIwL5mq/k1yOVnpbHg
9BIWnF7Cxl37ePLdbTy1ahsvf1hLVloy51eM4rPHjuHsqUWkp1iCMYmly6QiItNU9UMROTHUelVd
Fb2whqaOQpKTrObXkFE+Kpsb50zjexdO5W1/PX9ds51n1u7kz+9tJzs9hQtmjOJzx43hzMlFVkHB
JITuzlS+CywE/jvEOgXOi0pEQ5j/cCFJO1MZapKThDMmF3LG5EJunXsMb/rq+evq7Ty3bidPrtpG
TkYKF80YzcXHjub0skK7RGaGrC6TiqoudN/PHbhwhjZfoNEVkrR7Hoay1OQkzp5SxNlTirjt88fy
RnWAv67ewTNrd/KHlTVkpiVz9pQiLqgYxXnTRjIi02YCmqEjrCc/isjpfPrmx0ejFNOQ5Q80WSHJ
BJOWknR4enJzaxtv+ep5fv0uXly/i2fW7iQ5SZhVks8FFaO4oGKU3WRpBr1wZn/9Dq/U/XscuflR
VfWbUY4t6gZ69teFd77KhPxM7l9w8oAd08Sn9nZlzbYGXli/k+fX7WKjuyl22uhsVz6miJMm5tmN
liYu9Wv2F1AJVGgiP3c4Atralc31+zln6shYh2LiQFKSMHP8CGaOH8H3L5rG5romXli/ixc/2MX9
r/v57as+hqencMbkAs6ZOpKzpxQxdsSwWIdtTI/CSSprgdHAjt52LiJzgF8AycD9qnp7p/XpwKPA
SUA98CVV3ezW3QxcjXd29E1Vfa67PsW7m/DHwOVun9+o6i97G3O0dBSStKc9mlBKCrO45qxJXHPW
JPYdPMTfq+t59aMAr26o5bl1uwCYMmo450wdyVnlRVSW5Nlgv4lL4SSVQmC9iLyDdwMkAKp6aXc7
iUgycDdwAd6TI1eIyDJVXR+02dXAHlWdLCLzgDuAL4lIBd4TJmcAY4EXRWSK26erPr8CjAemqWq7
iMTVKUHHI4Qn2cwv04PsjFTmHDOaOceMRlXZWNvI3zbU8upHAR76+ybufc1PWkoSlRPzOL2sgNMn
F3LcuFxS7FKZiQPhJJUf9bHvWUB1xzPsRWQRMBcITipzg/pfCvzanXHMBRapajOwSUSqXX900+fX
gS+rajuAqtb2Me6o8Nl0YtMHIsKUUdlMGZXNwrPKaGpu5W1/PW/6vK+fPf8RPP8Rw9NTOKU0n9PK
CjhjciFTR2WTlGSlgMzA6zapuLONH/VxWvE4YGvQ6xrglK62UdVWEWkAClz72532HeeWu+qzDO8s
5/NAAO+S2cYQ72kh3v03TJgwoffvqo98ASskafovKz2F2dNHMXv6KMArGfOWr543fXW86avnpQ+9
v6UKstI4ZVI+J5d4X9PH5JBsScYMgG6Tiqq2iUi7iOSqasNABdVH6cBBVa0UkS8ADwKf6byRqt4L
3Ave7K+BCs4faLRy9ybi8rPS+OxxY/jscWMA2L73AG/56vm7r47l/t08/f5OAIanp3DixDxmleRx
ckk+x48fYWMyJirCufzVCLwvIi8ATR2NYUwp3oY3xtGh2LWF2qZGRFKAXLwB++727aq9BnjSLT8F
PNRDfAPKX9fEOVOKYh2GGeLGjhjGP51UzD+dVAx4SWbF5t3e16Y93uUyIC05ieOKc6ksyWdWaR4z
x+fZWbSJiHCSypMc+WXdGyuAchEpxfvFPw/4cqdtlgELgLeAy4CXVVVFZBnwuIj8HG+gvhx4B69C
cld9/gk4F9gEnA181IeYo6KjkKQN0puBNnbEMObO9MrzA+zd30LV5j2s2LybdzbvdtOXvRP2koJM
Zo4fwQkT8pg5fgTTx+TYjbqm18KpUvxIXzp2YyTXA8/hTf99UFXXicitQJWqLgMeAH7nBuJ34yUJ
3HZL8AbgW4HrVLUNIFSf7pC3A4+JyHfwzq6+2pe4o6GjkKRNJzaxNiIzjfMrRnF+hTcmc6CljdU1
e3lv617e27KXN331/Om97YBXDeDYcbku0Xj31IwbMcyeBWS6Fc4d9eXAfwEVQEZHu6pOim5o0TdQ
d9T/cWUN//6H1bz43bOZbM+mN3FMVdnRcJD3tu7l3S17eHfLXt7f1kBzazsARdnpHDcul2Pc17Hj
chmVk26JJsH09476h4AfAnfiXV66ErBz4l7w11khSTM4iAhjRwxj7IhhXHKsN/h/qK2dD3fs492t
e3jPJZlXNtTS7v4eLRyezjHjcjg2KNGMyc2wRJOgwkkqw1T1JRERVf0Y+JGIrARuiXJsQ4avtomJ
VkjSDFKpyUkcW5zLscW5/OtpXtv+llbWb/+EtdsaeH+b9/21jwKHE01BVhozxuVy7Lgcpo/JYdro
HEoKMu0GzQQQTlJpFpEkYKMbz9gG2DWcXvDXNdqDucyQkpmWQmVJPpUl+YfbDrS08cFOl2hqGnh/
WwO/ra6jzWWa9JQkpozKZtrobC/RjMlm+ugc8mzW2ZASTlL5FpAJfBP4v3iXwBZEM6ihpK1d2Vy3
n3OtkKQZ4oalJXPihDxOnJB3uO3goTaqaxv5cOc+PtzxCR/u3MfLH9byh5U1h7cZlZPOtNFHksy0
MdmUFQ23Cs2DVDizv1YAiEi7ql4Z/ZCGlpo9+2lpa7czFZOQMlKTDw/qBwvsa+bDnZ/w4Y59fOC+
v+Wrp6XNmxCQmiyUFmZRPjKbspHDKR85nPJRwyktzCI9xW7ajGc9JhUROQ1v6u9wYIKIHA98TVWv
jXZwQ8GR6cR2xdCYDkXZ6RRlF/GZ8iM3BB9qa2dTXRMfuDOajbsaWb/jE55Zu+PwWE2SwMSCLCYH
JZrJRdmUjcwiMy2sZw6aKPV0tq4AABOmSURBVAvnp3AXcBHejYqo6moROSuqUQ0hVp3YmPCkJicd
Lp45N6j94KE2NtU1sbG2kepd+9hY28jG2kZe+bCW1vYjt0QU5w2jfORwyoqGU1qURWlhFpMKh9uU
5wEWVmpX1a2dfihtXW1rjmaFJI3pn4zUZKaP8WaRBTvU1s7H9U1s3NV4ONFs3LWPN331h++rAchM
S6akIIvSoiwmFXrJpiPh5GamDvTbGfLCSSpb3TPqVURS8QbuP4huWEOHP9Bol76MiYLU5CQmj8xm
8shsLg5qb29XdnxykE2BJjbVNeKva2JTXRNrtzXwzPtHLqWBV5CzNCjRlBZmMSE/kwkFmeRkWMLp
i3CSyr/hPWlxHN504ucBG08Jky/QxLlTrZCkMQMlKUkYN2IY40YM48zywqPWtbS2s2X3fjbVeQln
k0s4r28MsDRoRhrAiMxUJuZnMj4/k4kFmUw4vJzF6JwMe5RAF8KZ/VUHXBHcJiLfxhtrMd1oOHCI
usZmyqw0izFxIS0lickjh7tySaOOWtfY3MqW+v1s2d3Elt37+bh+P1t27+f9bQ08u3bnUeM3aclJ
FOcNOyrhdJzhjBsxjOwEPsvp63SJ72JJpUf+jkF6e46KMXFveHoKFWNzqBib86l1rW3t7Gg4eFSy
6Ug+q7bsYd/B1qO2z8lIoTgvk3F53hlTcZ73NW6E15aXmTpkJw/0NakMzU8jwjqmE9vML2MGt5Tk
JMa7y19nTD56narScODQ4WSzbe8Btu05wLa9B9hSv583q+toajl6blNmWrJ3ia5TsinOG0bxiGEU
Dk8ftI+D7mtSGbAnJg5mvkAjKUnCxAIrJGnMUCUijMhMY0RmGsePH/Gp9R1Jp2bPAWpcsvGSzn5q
9hzgva172bv/0FH7pCUnMTo3g9G5GYzNzWB07jDGHH49jNG5GRRkpcVl4ukyqYjIPkInDwGGRS2i
IcQfaGJCfqaVmzAmgQUnnc6VBTo0Nreyfe8BavbsZ9ueA9TsPcDOhoPs2HuQlVv2sLNhB4fajv51
nJosjMo5kmQ6ks4Yl4DG5GbE5Iyny6SiqtkDGchQ5BWStEtfxpjuDU9POXzjZyjt7Up9U4uXaBoO
sPOTg2zfe5CdDQcOP//m2bUHD5e56ZCS5CWe0bkZjMpJ95ZzMhiVk8HpZQWMzMkIebz+sLoGUWKF
JI0xkZKUJK60TTrHFoc+21FVdje1sKPhIDsajiQcb/kgH+7cx6sbAofHdx69apYllcGko5Ck3fho
jBkIIkLB8HQKhqd3eZkNYN/BQ+z6pJkxuZFPKGBJJWqO1Pyy6cTGmPiRnZEa1ftoojqCLCJzRGSD
iFSLyE0h1qeLyGK3frmIlAStu9m1bxCRi3rR5y9FpDFa7ylcNp3YGJOIopZURCQZuBu4GKgA5otI
RafNrgb2qOpk4E7gDrdvBTAPmAHMAe4RkeSe+hSRSiCPOOALNJFnhSSNMQkmmmcqs4BqVfWraguw
CI6qaI17/YhbXgrMFu8207nAIlVtVtVNQLXrr8s+XcL5KXBDFN9T2HwBm/lljEk80Uwq44CtQa9r
XFvIbVS1FWgACrrZt7s+rweWqeqO7oISkYUiUiUiVYFAoFdvqDf8gSbKbDzFGJNghsRdeSIyFrgc
+FVP26rqvapaqaqVRUXRqR7cUUjSzlSMMYkmmkllGzA+6HWxawu5jYikALlAfTf7dtV+AjAZqBaR
zUCmiFRH6o30lhWSNMYkqmgmlRVAuYiUikga3sD7sk7bLAMWuOXLgJdVVV37PDc7rBQoB97pqk9V
/R9VHa2qJapaAux3g/8x4et4Lr2VvDfGJJio3aeiqq0icj3wHJAMPKiq60TkVqBKVZcBDwC/c2cV
u/GSBG67JcB6oBW4TlXbAEL1Ga330Fd+V0hyQr4VkjTGJJao3vyoqk8DT3dquyVo+SDeWEiofW8D
bgunzxDbxPQUwR9oYkKBFZI0xiQe+60XBb5AI5MK7dKXMSbxWFKJsNa2dj6u30/ZSBukN8YkHksq
EVaz54BXSNLOVIwxCciSSoT566yQpDEmcVlSibCOQpJW8t4Yk4gsqUSYL9BIXmYqeVZI0hiTgCyp
RJgv0GRnKcaYhGVJJcL8gUYbTzHGJCxLKhHUsP8QdY0tVkjSGJOwLKlEkM/N/LLLX8aYRGVJJYKO
PELYLn8ZYxKTJZUIskKSxphEZ0klgnyBRiskaYxJaPbbL4L8Np3YGJPgLKlESGtbO5vrm2w8xRiT
0CypREjNngMcalMrJGmMSWiWVCKko5Cklbw3xiQySyoR4qt104ntTMUYk8AsqUSIv66R/Kw0KyRp
jEloUU0qIjJHRDaISLWI3BRifbqILHbrl4tISdC6m137BhG5qKc+ReQx175WRB4UkdRovrfOfLVN
TCq0S1/GmMQWtaQiIsnA3cDFQAUwX0QqOm12NbBHVScDdwJ3uH0rgHnADGAOcI+IJPfQ52PANOBY
YBjw1Wi9t1D8dVZI0hhjonmmMguoVlW/qrYAi4C5nbaZCzzilpcCs0VEXPsiVW1W1U1Ateuvyz5V
9Wl1gHeA4ii+t6N0FJK0e1SMMYkumkllHLA16HWNawu5jaq2Ag1AQTf79tinu+z1L8Cz/X4HYfId
foSwJRVjTGIbigP19wCvqerroVaKyEIRqRKRqkAgEJEDHnmEsF3+MsYktmgmlW3A+KDXxa4t5DYi
kgLkAvXd7NttnyLyQ6AI+G5XQanqvapaqaqVRUVFvXxLoflcIcnxVkjSGJPgoplUVgDlIlIqIml4
A+/LOm2zDFjgli8DXnZjIsuAeW52WClQjjdO0mWfIvJV4CJgvqq2R/F9fYo/0MhEKyRpjDGkRKtj
VW0VkeuB54Bk4EFVXScitwJVqroMeAD4nYhUA7vxkgRuuyXAeqAVuE5V2wBC9ekO+VvgY+Atb6yf
J1X11mi9v2C+QJONpxhjDFFMKuDNyAKe7tR2S9DyQeDyLva9DbgtnD5de1TfS1da29r5uL6J2dNH
xuLwxhgTV+x6TT8dLiRpZyrGGGNJpb98gY7n0tvML2OMsaTST4efS2+FJI0xxpJKf/kCVkjSGGM6
WFLpJ3/ACkkaY0wHSyr95As02iC9McY4llT6oWH/IeqbWqw6sTHGOJZU+qGjkKSdqRhjjMeSSj/4
ajuqE9uZijHGgCWVfvHXNZGabIUkjTGmgyWVfvDVNjIh3wpJGmNMB/tt2A/+OiskaYwxwSyp9FFH
IUkbpDfGmCMsqfTRVldI0gbpjTHmCEsqfeQP2HRiY4zpzJJKH1l1YmOM+TRLKn3kDzRRkJXGiEwr
JGmMMR0sqfSRL9Bo4ynGGNOJJZU+8qoT23iKMcYEs6TSB3v3t1Df1ELZSDtTMcaYYFFNKiIyR0Q2
iEi1iNwUYn26iCx265eLSEnQuptd+wYRuainPkWk1PVR7fqM2mCHz572aIwxIUUtqYhIMnA3cDFQ
AcwXkYpOm10N7FHVycCdwB1u3wpgHjADmAPcIyLJPfR5B3Cn62uP6zsqDk8nHmlJxRhjgkXzTGUW
UK2qflVtARYBczttMxd4xC0vBWaLiLj2RararKqbgGrXX8g+3T7nuT5wff5jtN6YL+AKSeYNi9Yh
jDFmUIpmUhkHbA16XePaQm6jqq1AA1DQzb5dtRcAe10fXR0LABFZKCJVIlIVCAT68LagpCCTz58w
jhQrJGmMMUdJuN+KqnqvqlaqamVRUVGf+pg3awI/uez4CEdmjDGDXzSTyjZgfNDrYtcWchsRSQFy
gfpu9u2qvR4Y4fro6ljGGGOiLJpJZQVQ7mZlpeENvC/rtM0yYIFbvgx4WVXVtc9zs8NKgXLgna76
dPu84vrA9fnnKL43Y4wxIaT0vEnfqGqriFwPPAckAw+q6joRuRWoUtVlwAPA70SkGtiNlyRw2y0B
1gOtwHWq2gYQqk93yBuBRSLyY+Bd17cxxpgBJN4f+YmpsrJSq6qqYh2GMcYMKiKyUlUrQ61LuIF6
Y4wx0WNJxRhjTMRYUjHGGBMxllSMMcZETEIP1ItIAPi4j7sXAnURDCdSLK7esbh6x+LqnXiNC/oX
20RVDXn3eEInlf4QkaquZj/EksXVOxZX71hcvROvcUH0YrPLX8YYYyLGkooxxpiIsaTSd/fGOoAu
WFy9Y3H1jsXVO/EaF0QpNhtTMcYYEzF2pmKMMSZiLKkYY4yJGEsqfSAic0Rkg4hUi8hNA3C8zSLy
voi8JyJVri1fRF4QkY3ue55rFxH5pYttjYicGNTPArf9RhFZ0NXxeojlQRGpFZG1QW0Ri0VETnLv
tdrtK/2I60ciss19bu+JyCVB6252x9ggIhcFtYf82brHLSx37Yvdoxd6imm8iLwiIutFZJ2IfCse
Pq9u4orp5+X2yxCRd0RktYvtP7rrT7zHYyx27ctFpKSvMfcxrodFZFPQZzbTtQ/kv/1kEXlXRP4a
D58VqmpfvfjCK7nvAyYBacBqoCLKx9wMFHZq+wlwk1u+CbjDLV8CPAMIcCqw3LXnA373Pc8t5/Uh
lrOAE4G10YgF77k5p7p9ngEu7kdcPwK+F2LbCvdzSwdK3c8zubufLbAEmOeWfwt8PYyYxgAnuuVs
4CN37Jh+Xt3EFdPPy20rwHC3nAosd+8vZH/AtcBv3fI8YHFfY+5jXA8Dl4XYfiD/7X8XeBz4a3ef
/UB9Vnam0nuzgGpV9atqC7AImBuDOOYCj7jlR4B/DGp/VD1v4z0RcwxwEfCCqu5W1T3AC8Cc3h5U
VV/De/ZNxGNx63JU9W31/rU/GtRXX+Lqylxgkao2q+omoBrv5xryZ+v+YjwPWBriPXYX0w5VXeWW
9wEfAOOI8efVTVxdGZDPy8WjqtroXqa6L+2mv+DPcikw2x2/VzH3I66uDMjPUkSKgc8C97vX3X32
A/JZWVLpvXHA1qDXNXT/HzISFHheRFaKyELXNkpVd7jlncCoHuKLZtyRimWcW45kjNe7yw8PirvM
1Ie4CoC9qtra17jcpYYT8P7CjZvPq1NcEAefl7uc8x5Qi/dL19dNf4djcOsb3PEj/v+gc1yq2vGZ
3eY+sztFJL1zXGEev68/y7uAG4B297q7z35APitLKoPDmap6InAxcJ2InBW80v1lExdzw+MpFuA3
QBkwE9gB/HcsghCR4cAfgW+r6ifB62L5eYWIKy4+L1VtU9WZQDHeX8vTYhFHZ53jEpFjgJvx4jsZ
75LWjQMVj4h8DqhV1ZUDdcxwWFLpvW3A+KDXxa4talR1m/teCzyF9x9tlztlxn2v7SG+aMYdqVi2
ueWIxKiqu9wvgnbgPrzPrS9x1eNdvkjp1N4jEUnF+8X9mKo+6Zpj/nmFiisePq9gqroXeAU4rZv+
Dsfg1ue640ft/0FQXHPcpURV1WbgIfr+mfXlZ3kGcKmIbMa7NHUe8Ati/Vn1NOhiX58aFEvBG1wr
5cjg1YwoHi8LyA5afhNvLOSnHD3Y+xO3/FmOHiB8x7XnA5vwBgfz3HJ+H2Mq4egB8YjFwqcHKy/p
R1xjgpa/g3fdGGAGRw9M+vEGJbv82QJ/4OjBz2vDiEfwro3f1ak9pp9XN3HF9PNy2xYBI9zyMOB1
4HNd9Qdcx9GDz0v6GnMf4xoT9JneBdweo3/753BkoD62n1Vffqkk+hfezI6P8K71/iDKx5rkfpir
gXUdx8O7FvoSsBF4MegfpgB3u9jeByqD+roKbxCuGriyj/E8gXdp5BDeNdarIxkLUAmsdfv8Glf1
oY9x/c4ddw2wjKN/af7AHWMDQbNsuvrZup/DOy7ePwDpYcR0Jt6lrTXAe+7rklh/Xt3EFdPPy+13
HPCui2EtcEt3/QEZ7nW1Wz+przH3Ma6X3We2Fvg9R2aIDdi/fbfvORxJKjH9rKxMizHGmIixMRVj
jDERY0nFGGNMxFhSMcYYEzGWVIwxxkSMJRVjjDERY0nFmF4SkYKgqrQ75ejKvuFW431IRKb2M45J
IjKvP30YE2k2pdiYfhCRHwGNqvqzTu2C9/+rPeSOkTn2+cD1qhpWsUZjBoKdqRgTISIyWbxnlDyG
d6PqGBG5V0Sq3DM4bgna9g0RmSkiKSKyV0RuF+9ZHW+JyMgQfZ/n1r8nIqtEJAu4HTjXtX3T9fVz
8Z77sUZEvur2PV+856c8456NcXe4z+owprcsqRgTWdOAO1W1Qr2abTepaiVwPHCBiFSE2CcXeFVV
jwfewrvjurPvAwvVK2h4FnAQr8TLK6o6U1V/CSzEKzA4C6/A4XUiMsHtfwrwdbxnZ0wnNo9rMAnA
kooxkeVT1aqg1/NFZBWwCu+XeaikckBVn3HLK/FqmHX2d+AXIvINvOdutIXY5kLgSleefTkwAih3
695W1c1uv0V4pVqMibiUnjcxxvRCU8eCiJQD3wJmqepeEfk9Xv2lzlqCltsI8f9SVX8sIsvwChW+
LSKzQ/QjeMUDXzqq0Rt76Tx4aoOpJirsTMWY6MkB9gGfBD31r09EpExV16jqf+Gd9Ux1fWcHbfYc
cG1H2XMRmSoiw9y6U0VkgogkA18E3uhrLMZ0x85UjImeVcB64EPgY7xLWH31PRH5DN4T/tYAz7v2
ZBFZDTyAVxV3AvCeG4ev5cjYyTt4ZdDL8CojL+tHLMZ0yaYUGzPE2dRjM5Ds8pcxxpiIsTMVY4wx
EWNnKsYYYyLGkooxxpiIsaRijDEmYiypGGOMiRhLKsYYYyLm/wNIQcwJUvgs7QAAAABJRU5ErkJg
gg==
"
>
</div>

</div>

</div>
</div>

</div>
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
</div><div class="inner_cell">
<div class="text_cell_render border-box-sizing rendered_html">
<h3 id="Loss">Loss<a class="anchor-link" href="#Loss">&#182;</a></h3>
</div>
</div>
</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="prompt input_prompt">In&nbsp;[&nbsp;]:</div>
<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="n">loss_object</span> <span class="o">=</span> <span class="n">keras</span><span class="o">.</span><span class="n">losses</span><span class="o">.</span><span class="n">SparseCategoricalCrossentropy</span><span class="p">(</span><span class="n">from_logits</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">reduction</span><span class="o">=</span><span class="s1">&#39;none&#39;</span><span class="p">)</span>

<span class="k">def</span> <span class="nf">loss_function</span><span class="p">(</span><span class="n">real</span><span class="p">,</span> <span class="n">pred</span><span class="p">):</span>
    <span class="c1"># 去除padding，去噪声</span>
    <span class="n">mask</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">math</span><span class="o">.</span><span class="n">logical_not</span><span class="p">(</span><span class="n">tf</span><span class="o">.</span><span class="n">math</span><span class="o">.</span><span class="n">equal</span><span class="p">(</span><span class="n">real</span><span class="p">,</span> <span class="mi">0</span><span class="p">))</span>
    <span class="n">loss_</span> <span class="o">=</span> <span class="n">loss_object</span><span class="p">(</span><span class="n">real</span><span class="p">,</span> <span class="n">pred</span><span class="p">)</span>
    
    <span class="n">mask</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">cast</span><span class="p">(</span><span class="n">mask</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="n">loss_</span><span class="o">.</span><span class="n">dtype</span><span class="p">)</span>
    <span class="n">loss_</span> <span class="o">*=</span> <span class="n">mask</span>
    <span class="k">return</span> <span class="n">tf</span><span class="o">.</span><span class="n">reduce_mean</span><span class="p">(</span><span class="n">loss_</span><span class="p">)</span>
</pre></div>

    </div>
</div>
</div>

</div>
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
</div><div class="inner_cell">
<div class="text_cell_render border-box-sizing rendered_html">
<h3 id="&#24037;&#20855;&#20989;&#25968;&#65306;mask&#21019;&#24314;">&#24037;&#20855;&#20989;&#25968;&#65306;mask&#21019;&#24314;<a class="anchor-link" href="#&#24037;&#20855;&#20989;&#25968;&#65306;mask&#21019;&#24314;">&#182;</a></h3><p>问题：一个多头注意力，只能有一个mask，但是DecoderLayer上有两个mask，即look_ahead_mask,decoder_padding_mask，该怎么办？</p>
<p>答：将两个mask做与操作，只要其中一个有mask，就mask</p>

</div>
</div>
</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="prompt input_prompt">In&nbsp;[&nbsp;]:</div>
<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="k">def</span> <span class="nf">create_masks</span><span class="p">(</span><span class="n">inp</span><span class="p">,</span> <span class="n">tar</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Encoder:</span>
<span class="sd">      - encoder_padding_mask (self attention of EncoderLayer)</span>
<span class="sd">    Decoder:</span>
<span class="sd">      - look_ahead_mask (self attention of DecoderLayer)</span>
<span class="sd">      - encoder_decoder_padding_mask (encoder-decoder attention of DecoderLayer)</span>
<span class="sd">      - decoder_padding_mask (self attention of DecoderLayer)</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="n">encoder_padding_mask</span> <span class="o">=</span> <span class="n">create_padding_mask</span><span class="p">(</span><span class="n">inp</span><span class="p">)</span>
    <span class="n">encoder_decoder_padding_mask</span> <span class="o">=</span> <span class="n">create_padding_mask</span><span class="p">(</span><span class="n">inp</span><span class="p">)</span>
    
    <span class="n">look_ahead_mask</span> <span class="o">=</span> <span class="n">create_look_ahead_mask</span><span class="p">(</span><span class="n">tf</span><span class="o">.</span><span class="n">shape</span><span class="p">(</span><span class="n">tar</span><span class="p">)[</span><span class="mi">1</span><span class="p">])</span>
    <span class="n">decoder_padding_mask</span> <span class="o">=</span> <span class="n">create_padding_mask</span><span class="p">(</span><span class="n">tar</span><span class="p">)</span>
    <span class="n">decoder_mask</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">maximum</span><span class="p">(</span><span class="n">decoder_padding_mask</span><span class="p">,</span> <span class="n">look_ahead_mask</span><span class="p">)</span>
    
<span class="c1">#     print(encoder_padding_mask.shape)</span>
<span class="c1">#     print(encoder_decoder_padding_mask.shape)</span>
<span class="c1">#     print(look_ahead_mask.shape)</span>
<span class="c1">#     print(decoder_padding_mask.shape)</span>
<span class="c1">#     print(decoder_mask.shape)</span>
    
    <span class="k">return</span> <span class="n">encoder_padding_mask</span><span class="p">,</span> <span class="n">decoder_mask</span><span class="p">,</span> <span class="n">encoder_decoder_padding_mask</span>
</pre></div>

    </div>
</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="prompt input_prompt">In&nbsp;[50]:</div>
<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="c1"># test </span>
<span class="n">temp_inp</span><span class="p">,</span> <span class="n">temp_tar</span> <span class="o">=</span> <span class="nb">iter</span><span class="p">(</span><span class="n">train_dataset</span><span class="o">.</span><span class="n">take</span><span class="p">(</span><span class="mi">1</span><span class="p">))</span><span class="o">.</span><span class="n">next</span><span class="p">()</span>

<span class="nb">print</span><span class="p">(</span><span class="n">temp_inp</span><span class="o">.</span><span class="n">shape</span><span class="p">)</span>
<span class="nb">print</span><span class="p">(</span><span class="n">temp_tar</span><span class="o">.</span><span class="n">shape</span><span class="p">)</span>
<span class="n">create_masks</span><span class="p">(</span><span class="n">temp_inp</span><span class="p">,</span> <span class="n">temp_tar</span><span class="p">)</span>
</pre></div>

    </div>
</div>
</div>

<div class="output_wrapper">
<div class="output">


<div class="output_area">

    <div class="prompt"></div>


<div class="output_subarea output_stream output_stdout output_text">
<pre>(64, 38)
(64, 37)
</pre>
</div>
</div>

<div class="output_area">

    <div class="prompt output_prompt">Out[50]:</div>




<div class="output_text output_subarea output_execute_result">
<pre>(&lt;tf.Tensor: shape=(64, 1, 1, 38), dtype=float32, numpy=
 array([[[[0., 0., 0., ..., 1., 1., 1.]]],
 
 
        [[[0., 0., 0., ..., 1., 1., 1.]]],
 
 
        [[[0., 0., 0., ..., 1., 1., 1.]]],
 
 
        ...,
 
 
        [[[0., 0., 0., ..., 1., 1., 1.]]],
 
 
        [[[0., 0., 0., ..., 1., 1., 1.]]],
 
 
        [[[0., 0., 0., ..., 1., 1., 1.]]]], dtype=float32)&gt;,
 &lt;tf.Tensor: shape=(64, 1, 37, 37), dtype=float32, numpy=
 array([[[[0., 1., 1., ..., 1., 1., 1.],
          [0., 0., 1., ..., 1., 1., 1.],
          [0., 0., 0., ..., 1., 1., 1.],
          ...,
          [0., 0., 0., ..., 1., 1., 1.],
          [0., 0., 0., ..., 1., 1., 1.],
          [0., 0., 0., ..., 1., 1., 1.]]],
 
 
        [[[0., 1., 1., ..., 1., 1., 1.],
          [0., 0., 1., ..., 1., 1., 1.],
          [0., 0., 0., ..., 1., 1., 1.],
          ...,
          [0., 0., 0., ..., 1., 1., 1.],
          [0., 0., 0., ..., 1., 1., 1.],
          [0., 0., 0., ..., 1., 1., 1.]]],
 
 
        [[[0., 1., 1., ..., 1., 1., 1.],
          [0., 0., 1., ..., 1., 1., 1.],
          [0., 0., 0., ..., 1., 1., 1.],
          ...,
          [0., 0., 0., ..., 1., 1., 1.],
          [0., 0., 0., ..., 1., 1., 1.],
          [0., 0., 0., ..., 1., 1., 1.]]],
 
 
        ...,
 
 
        [[[0., 1., 1., ..., 1., 1., 1.],
          [0., 0., 1., ..., 1., 1., 1.],
          [0., 0., 0., ..., 1., 1., 1.],
          ...,
          [0., 0., 0., ..., 1., 1., 1.],
          [0., 0., 0., ..., 1., 1., 1.],
          [0., 0., 0., ..., 1., 1., 1.]]],
 
 
        [[[0., 1., 1., ..., 1., 1., 1.],
          [0., 0., 1., ..., 1., 1., 1.],
          [0., 0., 0., ..., 1., 1., 1.],
          ...,
          [0., 0., 0., ..., 1., 1., 1.],
          [0., 0., 0., ..., 1., 1., 1.],
          [0., 0., 0., ..., 1., 1., 1.]]],
 
 
        [[[0., 1., 1., ..., 1., 1., 1.],
          [0., 0., 1., ..., 1., 1., 1.],
          [0., 0., 0., ..., 1., 1., 1.],
          ...,
          [0., 0., 0., ..., 1., 1., 1.],
          [0., 0., 0., ..., 1., 1., 1.],
          [0., 0., 0., ..., 1., 1., 1.]]]], dtype=float32)&gt;,
 &lt;tf.Tensor: shape=(64, 1, 1, 38), dtype=float32, numpy=
 array([[[[0., 0., 0., ..., 1., 1., 1.]]],
 
 
        [[[0., 0., 0., ..., 1., 1., 1.]]],
 
 
        [[[0., 0., 0., ..., 1., 1., 1.]]],
 
 
        ...,
 
 
        [[[0., 0., 0., ..., 1., 1., 1.]]],
 
 
        [[[0., 0., 0., ..., 1., 1., 1.]]],
 
 
        [[[0., 0., 0., ..., 1., 1., 1.]]]], dtype=float32)&gt;)</pre>
</div>

</div>

</div>
</div>

</div>
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
</div><div class="inner_cell">
<div class="text_cell_render border-box-sizing rendered_html">
<h3 id="&#27169;&#22411;&#20445;&#23384;">&#27169;&#22411;&#20445;&#23384;<a class="anchor-link" href="#&#27169;&#22411;&#20445;&#23384;">&#182;</a></h3>
</div>
</div>
</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="prompt input_prompt">In&nbsp;[&nbsp;]:</div>
<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="n">checkpoint_path</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="s1">&#39;/content/drive/My Drive/Colab Notebooks/transformer&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">checkpoint_path</span><span class="p">):</span>
    <span class="n">os</span><span class="o">.</span><span class="n">mkdir</span><span class="p">(</span><span class="n">checkpoint_path</span><span class="p">)</span>
    <span class="nb">print</span><span class="p">(</span><span class="n">checkpoint_path</span><span class="p">)</span>
<span class="n">ckpt</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">train</span><span class="o">.</span><span class="n">Checkpoint</span><span class="p">(</span><span class="n">transformer</span><span class="o">=</span><span class="n">transformer</span><span class="p">,</span> <span class="n">optimizer</span><span class="o">=</span><span class="n">optimizer</span><span class="p">)</span>
<span class="n">ckpt_manager</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">train</span><span class="o">.</span><span class="n">CheckpointManager</span><span class="p">(</span><span class="n">ckpt</span><span class="p">,</span> <span class="n">checkpoint_path</span><span class="p">,</span> <span class="n">max_to_keep</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>

<span class="c1"># 如果检查点存在，则恢复最新的检查点</span>
<span class="k">if</span> <span class="n">ckpt_manager</span><span class="o">.</span><span class="n">latest_checkpoint</span><span class="p">:</span>
    <span class="n">ckpt</span><span class="o">.</span><span class="n">restore</span><span class="p">(</span><span class="n">ckpt_manager</span><span class="o">.</span><span class="n">latest_checkpoint</span><span class="p">)</span>
    <span class="nb">print</span><span class="p">(</span><span class="s1">&#39;Latest checkpoint restored!&#39;</span><span class="p">)</span>
</pre></div>

    </div>
</div>
</div>

</div>
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
</div><div class="inner_cell">
<div class="text_cell_render border-box-sizing rendered_html">
<h3 id="&#35757;&#32451;Train-step">&#35757;&#32451;Train step<a class="anchor-link" href="#&#35757;&#32451;Train-step">&#182;</a></h3>
</div>
</div>
</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="prompt input_prompt">In&nbsp;[&nbsp;]:</div>
<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="c1"># 方便可视训练过程，不是真正的NMT的训练准确度</span>
<span class="n">train_loss</span> <span class="o">=</span> <span class="n">keras</span><span class="o">.</span><span class="n">metrics</span><span class="o">.</span><span class="n">Mean</span><span class="p">(</span><span class="n">name</span> <span class="o">=</span> <span class="s1">&#39;train_loss&#39;</span><span class="p">)</span>
<span class="n">train_accuracy</span> <span class="o">=</span> <span class="n">keras</span><span class="o">.</span><span class="n">metrics</span><span class="o">.</span><span class="n">SparseCategoricalAccuracy</span><span class="p">(</span><span class="n">name</span> <span class="o">=</span> <span class="s1">&#39;train_accuracy&#39;</span><span class="p">)</span>

<span class="nd">@tf</span><span class="o">.</span><span class="n">function</span><span class="p">(</span><span class="n">experimental_relax_shapes</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">train_step</span><span class="p">(</span><span class="n">inp</span><span class="p">,</span> <span class="n">tar</span><span class="p">):</span>
    <span class="c1"># 把目标数据切分成decoder input和decoder output</span>
    <span class="n">tar_inp</span> <span class="o">=</span> <span class="n">tar</span><span class="p">[:,</span> <span class="p">:</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
    <span class="n">tar_real</span> <span class="o">=</span> <span class="n">tar</span><span class="p">[:,</span> <span class="mi">1</span><span class="p">:]</span>
    <span class="c1"># 获取mask</span>
    <span class="n">encoder_padding_mask</span><span class="p">,</span> <span class="n">decoder_mask</span><span class="p">,</span> <span class="n">encoder_decoder_padding_mask</span> <span class="o">=</span> <span class="n">create_masks</span><span class="p">(</span><span class="n">inp</span><span class="p">,</span> <span class="n">tar_inp</span><span class="p">)</span>
    
    <span class="c1"># 计算梯度</span>
    <span class="k">with</span> <span class="n">tf</span><span class="o">.</span><span class="n">GradientTape</span><span class="p">()</span> <span class="k">as</span> <span class="n">tape</span><span class="p">:</span>
        <span class="n">predictions</span><span class="p">,</span> <span class="n">_</span> <span class="o">=</span> <span class="n">transformer</span><span class="p">(</span><span class="n">inp</span><span class="p">,</span> <span class="n">tar_inp</span><span class="p">,</span> <span class="kc">True</span><span class="p">,</span> <span class="n">encoder_padding_mask</span><span class="p">,</span>
                                     <span class="n">decoder_mask</span><span class="p">,</span> <span class="n">encoder_decoder_padding_mask</span><span class="p">)</span>
        <span class="n">loss</span> <span class="o">=</span> <span class="n">loss_function</span><span class="p">(</span><span class="n">tar_real</span><span class="p">,</span> <span class="n">predictions</span><span class="p">)</span>
        
    <span class="n">gradients</span> <span class="o">=</span> <span class="n">tape</span><span class="o">.</span><span class="n">gradient</span><span class="p">(</span><span class="n">loss</span><span class="p">,</span> <span class="n">transformer</span><span class="o">.</span><span class="n">trainable_variables</span><span class="p">)</span>
    <span class="n">optimizer</span><span class="o">.</span><span class="n">apply_gradients</span><span class="p">(</span><span class="nb">zip</span><span class="p">(</span><span class="n">gradients</span><span class="p">,</span> <span class="n">transformer</span><span class="o">.</span><span class="n">trainable_variables</span><span class="p">))</span>
    
    <span class="c1"># 累积loss 和 accuracy</span>
    <span class="n">train_loss</span><span class="p">(</span><span class="n">loss</span><span class="p">)</span>
    <span class="n">train_accuracy</span><span class="p">(</span><span class="n">tar_real</span><span class="p">,</span> <span class="n">predictions</span><span class="p">)</span>
</pre></div>

    </div>
</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="prompt input_prompt">In&nbsp;[53]:</div>
<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="c1"># 训练：遍历数据集</span>
<span class="n">epochs</span> <span class="o">=</span> <span class="mi">20</span>
<span class="k">for</span> <span class="n">epoch</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">epochs</span><span class="p">):</span>
    <span class="n">start</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">()</span>
    <span class="n">train_loss</span><span class="o">.</span><span class="n">reset_states</span><span class="p">()</span>
    <span class="n">train_accuracy</span><span class="o">.</span><span class="n">reset_states</span><span class="p">()</span>
    
    <span class="c1"># 训练</span>
    <span class="k">for</span> <span class="p">(</span><span class="n">batch</span><span class="p">,</span> <span class="p">(</span><span class="n">inp</span><span class="p">,</span> <span class="n">tar</span><span class="p">))</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">train_dataset</span><span class="p">):</span>
        <span class="n">train_step</span><span class="p">(</span><span class="n">inp</span><span class="p">,</span> <span class="n">tar</span><span class="p">)</span>
        <span class="k">if</span> <span class="n">batch</span> <span class="o">%</span> <span class="mi">100</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
            <span class="nb">print</span><span class="p">(</span><span class="s1">&#39;Epoch </span><span class="si">{}</span><span class="s1"> Batch </span><span class="si">{}</span><span class="s1"> Loss </span><span class="si">{:.4f}</span><span class="s1"> Accuracy </span><span class="si">{:.4f}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
                <span class="n">epoch</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span> <span class="n">batch</span><span class="p">,</span> <span class="n">train_loss</span><span class="o">.</span><span class="n">result</span><span class="p">(),</span> <span class="n">train_accuracy</span><span class="o">.</span><span class="n">result</span><span class="p">()))</span>
    
    <span class="c1"># 保存</span>
    <span class="k">if</span><span class="p">(</span><span class="n">epoch</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)</span> <span class="o">%</span> <span class="mi">5</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
        <span class="n">ckpt_save_path</span> <span class="o">=</span> <span class="n">ckpt_manager</span><span class="o">.</span><span class="n">save</span><span class="p">()</span>
        <span class="nb">print</span><span class="p">(</span><span class="s1">&#39;Saving checkpoint for epoch </span><span class="si">{}</span><span class="s1"> at </span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">epoch</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span> <span class="n">ckpt_save_path</span><span class="p">))</span>
    
    <span class="c1"># 打印日志</span>
    <span class="nb">print</span><span class="p">(</span><span class="s1">&#39;Epoch </span><span class="si">{}</span><span class="s1"> Loss </span><span class="si">{:.4f}</span><span class="s1"> Accuracy </span><span class="si">{:.4f}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
        <span class="n">epoch</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span> <span class="n">train_loss</span><span class="o">.</span><span class="n">result</span><span class="p">(),</span> <span class="n">train_accuracy</span><span class="o">.</span><span class="n">result</span><span class="p">()))</span>
    <span class="nb">print</span><span class="p">(</span><span class="s1">&#39;Time taken for 1 epoch: </span><span class="si">{}</span><span class="s1"> secs</span><span class="se">\n</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">()</span> <span class="o">-</span> <span class="n">start</span><span class="p">))</span>
</pre></div>

    </div>
</div>
</div>

<div class="output_wrapper">
<div class="output">


<div class="output_area">

    <div class="prompt"></div>


<div class="output_subarea output_stream output_stdout output_text">
<pre>Epoch 1 Batch 0 Loss 3.9077 Accuracy 0.0000
Epoch 1 Batch 100 Loss 4.1565 Accuracy 0.0088
Epoch 1 Batch 200 Loss 4.0654 Accuracy 0.0167
Epoch 1 Batch 300 Loss 3.9126 Accuracy 0.0227
Epoch 1 Batch 400 Loss 3.7483 Accuracy 0.0299
Epoch 1 Batch 500 Loss 3.6023 Accuracy 0.0363
Epoch 1 Batch 600 Loss 3.4824 Accuracy 0.0434
Epoch 1 Batch 700 Loss 3.3732 Accuracy 0.0503
Epoch 1 Loss 3.3711 Accuracy 0.0504
Time taken for 1 epoch: 110.85187244415283 secs

Epoch 2 Batch 0 Loss 2.5038 Accuracy 0.0909
Epoch 2 Batch 100 Loss 2.5583 Accuracy 0.1050
Epoch 2 Batch 200 Loss 2.5285 Accuracy 0.1106
Epoch 2 Batch 300 Loss 2.4815 Accuracy 0.1145
Epoch 2 Batch 400 Loss 2.4513 Accuracy 0.1181
Epoch 2 Batch 500 Loss 2.4194 Accuracy 0.1210
Epoch 2 Batch 600 Loss 2.3962 Accuracy 0.1236
Epoch 2 Batch 700 Loss 2.3721 Accuracy 0.1260
Epoch 2 Loss 2.3713 Accuracy 0.1260
Time taken for 1 epoch: 83.61259198188782 secs

Epoch 3 Batch 0 Loss 2.3392 Accuracy 0.1427
Epoch 3 Batch 100 Loss 2.1646 Accuracy 0.1451
Epoch 3 Batch 200 Loss 2.1446 Accuracy 0.1452
Epoch 3 Batch 300 Loss 2.1355 Accuracy 0.1462
Epoch 3 Batch 400 Loss 2.1233 Accuracy 0.1480
Epoch 3 Batch 500 Loss 2.1150 Accuracy 0.1497
Epoch 3 Batch 600 Loss 2.1011 Accuracy 0.1516
Epoch 3 Batch 700 Loss 2.0881 Accuracy 0.1535
Epoch 3 Loss 2.0880 Accuracy 0.1536
Time taken for 1 epoch: 84.19668126106262 secs

Epoch 4 Batch 0 Loss 1.9075 Accuracy 0.1683
Epoch 4 Batch 100 Loss 1.9249 Accuracy 0.1725
Epoch 4 Batch 200 Loss 1.9139 Accuracy 0.1727
Epoch 4 Batch 300 Loss 1.8970 Accuracy 0.1750
Epoch 4 Batch 400 Loss 1.8824 Accuracy 0.1771
Epoch 4 Batch 500 Loss 1.8659 Accuracy 0.1788
Epoch 4 Batch 600 Loss 1.8539 Accuracy 0.1807
Epoch 4 Batch 700 Loss 1.8441 Accuracy 0.1828
Epoch 4 Loss 1.8437 Accuracy 0.1828
Time taken for 1 epoch: 84.03117537498474 secs

Epoch 5 Batch 0 Loss 1.5299 Accuracy 0.1867
Epoch 5 Batch 100 Loss 1.6641 Accuracy 0.2011
Epoch 5 Batch 200 Loss 1.6676 Accuracy 0.2025
Epoch 5 Batch 300 Loss 1.6584 Accuracy 0.2036
Epoch 5 Batch 400 Loss 1.6497 Accuracy 0.2048
Epoch 5 Batch 500 Loss 1.6403 Accuracy 0.2059
Epoch 5 Batch 600 Loss 1.6331 Accuracy 0.2070
Epoch 5 Batch 700 Loss 1.6278 Accuracy 0.2084
Saving checkpoint for epoch 5 at /content/drive/My Drive/Colab Notebooks/transformer/ckpt-1
Epoch 5 Loss 1.6283 Accuracy 0.2085
Time taken for 1 epoch: 85.19561910629272 secs

Epoch 6 Batch 0 Loss 1.4623 Accuracy 0.2297
Epoch 6 Batch 100 Loss 1.4823 Accuracy 0.2238
Epoch 6 Batch 200 Loss 1.4701 Accuracy 0.2235
Epoch 6 Batch 300 Loss 1.4676 Accuracy 0.2247
Epoch 6 Batch 400 Loss 1.4632 Accuracy 0.2258
Epoch 6 Batch 500 Loss 1.4570 Accuracy 0.2263
Epoch 6 Batch 600 Loss 1.4522 Accuracy 0.2271
Epoch 6 Batch 700 Loss 1.4439 Accuracy 0.2281
Epoch 6 Loss 1.4435 Accuracy 0.2281
Time taken for 1 epoch: 85.99724531173706 secs

Epoch 7 Batch 0 Loss 1.2713 Accuracy 0.2430
Epoch 7 Batch 100 Loss 1.2860 Accuracy 0.2443
Epoch 7 Batch 200 Loss 1.2951 Accuracy 0.2459
Epoch 7 Batch 300 Loss 1.2906 Accuracy 0.2469
Epoch 7 Batch 400 Loss 1.2810 Accuracy 0.2477
Epoch 7 Batch 500 Loss 1.2771 Accuracy 0.2482
Epoch 7 Batch 600 Loss 1.2740 Accuracy 0.2487
Epoch 7 Batch 700 Loss 1.2660 Accuracy 0.2492
Epoch 7 Loss 1.2657 Accuracy 0.2492
Time taken for 1 epoch: 85.94596433639526 secs

Epoch 8 Batch 0 Loss 1.1447 Accuracy 0.2524
Epoch 8 Batch 100 Loss 1.1042 Accuracy 0.2635
Epoch 8 Batch 200 Loss 1.1116 Accuracy 0.2647
Epoch 8 Batch 300 Loss 1.1162 Accuracy 0.2643
Epoch 8 Batch 400 Loss 1.1156 Accuracy 0.2651
Epoch 8 Batch 500 Loss 1.1157 Accuracy 0.2659
Epoch 8 Batch 600 Loss 1.1162 Accuracy 0.2662
Epoch 8 Batch 700 Loss 1.1157 Accuracy 0.2665
Epoch 8 Loss 1.1163 Accuracy 0.2666
Time taken for 1 epoch: 87.58165216445923 secs

Epoch 9 Batch 0 Loss 0.8514 Accuracy 0.3019
Epoch 9 Batch 100 Loss 0.9911 Accuracy 0.2827
Epoch 9 Batch 200 Loss 0.9955 Accuracy 0.2808
Epoch 9 Batch 300 Loss 1.0059 Accuracy 0.2818
Epoch 9 Batch 400 Loss 1.0070 Accuracy 0.2815
Epoch 9 Batch 500 Loss 1.0072 Accuracy 0.2814
Epoch 9 Batch 600 Loss 1.0100 Accuracy 0.2811
Epoch 9 Batch 700 Loss 1.0112 Accuracy 0.2808
Epoch 9 Loss 1.0117 Accuracy 0.2808
Time taken for 1 epoch: 87.4095528125763 secs

Epoch 10 Batch 0 Loss 1.0691 Accuracy 0.2939
Epoch 10 Batch 100 Loss 0.8989 Accuracy 0.2905
Epoch 10 Batch 200 Loss 0.9048 Accuracy 0.2909
Epoch 10 Batch 300 Loss 0.9113 Accuracy 0.2904
Epoch 10 Batch 400 Loss 0.9154 Accuracy 0.2905
Epoch 10 Batch 500 Loss 0.9202 Accuracy 0.2911
Epoch 10 Batch 600 Loss 0.9232 Accuracy 0.2908
Epoch 10 Batch 700 Loss 0.9274 Accuracy 0.2910
Saving checkpoint for epoch 10 at /content/drive/My Drive/Colab Notebooks/transformer/ckpt-2
Epoch 10 Loss 0.9275 Accuracy 0.2909
Time taken for 1 epoch: 87.70754671096802 secs

Epoch 11 Batch 0 Loss 0.8716 Accuracy 0.3028
Epoch 11 Batch 100 Loss 0.8232 Accuracy 0.3009
Epoch 11 Batch 200 Loss 0.8326 Accuracy 0.3014
Epoch 11 Batch 300 Loss 0.8420 Accuracy 0.3007
Epoch 11 Batch 400 Loss 0.8467 Accuracy 0.3004
Epoch 11 Batch 500 Loss 0.8537 Accuracy 0.3010
Epoch 11 Batch 600 Loss 0.8577 Accuracy 0.3001
Epoch 11 Batch 700 Loss 0.8606 Accuracy 0.2995
Epoch 11 Loss 0.8611 Accuracy 0.2995
Time taken for 1 epoch: 86.9194552898407 secs

Epoch 12 Batch 0 Loss 0.7313 Accuracy 0.3029
Epoch 12 Batch 100 Loss 0.7818 Accuracy 0.3110
Epoch 12 Batch 200 Loss 0.7859 Accuracy 0.3098
Epoch 12 Batch 300 Loss 0.7890 Accuracy 0.3091
Epoch 12 Batch 400 Loss 0.7948 Accuracy 0.3089
Epoch 12 Batch 500 Loss 0.7989 Accuracy 0.3081
Epoch 12 Batch 600 Loss 0.8039 Accuracy 0.3077
Epoch 12 Batch 700 Loss 0.8075 Accuracy 0.3073
Epoch 12 Loss 0.8078 Accuracy 0.3073
Time taken for 1 epoch: 86.03745102882385 secs

Epoch 13 Batch 0 Loss 0.7015 Accuracy 0.3045
Epoch 13 Batch 100 Loss 0.7318 Accuracy 0.3197
Epoch 13 Batch 200 Loss 0.7376 Accuracy 0.3187
Epoch 13 Batch 300 Loss 0.7422 Accuracy 0.3174
Epoch 13 Batch 400 Loss 0.7475 Accuracy 0.3159
Epoch 13 Batch 500 Loss 0.7497 Accuracy 0.3153
Epoch 13 Batch 600 Loss 0.7568 Accuracy 0.3154
Epoch 13 Batch 700 Loss 0.7615 Accuracy 0.3142
Epoch 13 Loss 0.7611 Accuracy 0.3142
Time taken for 1 epoch: 86.3356282711029 secs

Epoch 14 Batch 0 Loss 0.6588 Accuracy 0.3146
Epoch 14 Batch 100 Loss 0.6804 Accuracy 0.3223
Epoch 14 Batch 200 Loss 0.6888 Accuracy 0.3218
Epoch 14 Batch 300 Loss 0.6982 Accuracy 0.3209
Epoch 14 Batch 400 Loss 0.7071 Accuracy 0.3209
Epoch 14 Batch 500 Loss 0.7114 Accuracy 0.3204
Epoch 14 Batch 600 Loss 0.7157 Accuracy 0.3195
Epoch 14 Batch 700 Loss 0.7201 Accuracy 0.3188
Epoch 14 Loss 0.7200 Accuracy 0.3188
Time taken for 1 epoch: 86.5377869606018 secs

Epoch 15 Batch 0 Loss 0.6337 Accuracy 0.3730
Epoch 15 Batch 100 Loss 0.6401 Accuracy 0.3297
Epoch 15 Batch 200 Loss 0.6548 Accuracy 0.3280
Epoch 15 Batch 300 Loss 0.6631 Accuracy 0.3268
Epoch 15 Batch 400 Loss 0.6693 Accuracy 0.3264
Epoch 15 Batch 500 Loss 0.6729 Accuracy 0.3256
Epoch 15 Batch 600 Loss 0.6774 Accuracy 0.3245
Epoch 15 Batch 700 Loss 0.6831 Accuracy 0.3238
Saving checkpoint for epoch 15 at /content/drive/My Drive/Colab Notebooks/transformer/ckpt-3
Epoch 15 Loss 0.6833 Accuracy 0.3239
Time taken for 1 epoch: 87.99038195610046 secs

Epoch 16 Batch 0 Loss 0.5221 Accuracy 0.3413
Epoch 16 Batch 100 Loss 0.6207 Accuracy 0.3359
Epoch 16 Batch 200 Loss 0.6226 Accuracy 0.3323
Epoch 16 Batch 300 Loss 0.6311 Accuracy 0.3314
Epoch 16 Batch 400 Loss 0.6369 Accuracy 0.3308
Epoch 16 Batch 500 Loss 0.6403 Accuracy 0.3294
Epoch 16 Batch 600 Loss 0.6474 Accuracy 0.3292
Epoch 16 Batch 700 Loss 0.6535 Accuracy 0.3285
Epoch 16 Loss 0.6534 Accuracy 0.3285
Time taken for 1 epoch: 88.45781326293945 secs

Epoch 17 Batch 0 Loss 0.6035 Accuracy 0.3351
Epoch 17 Batch 100 Loss 0.5810 Accuracy 0.3387
Epoch 17 Batch 200 Loss 0.5946 Accuracy 0.3372
Epoch 17 Batch 300 Loss 0.6019 Accuracy 0.3356
Epoch 17 Batch 400 Loss 0.6060 Accuracy 0.3347
Epoch 17 Batch 500 Loss 0.6128 Accuracy 0.3343
Epoch 17 Batch 600 Loss 0.6184 Accuracy 0.3336
Epoch 17 Batch 700 Loss 0.6237 Accuracy 0.3328
Epoch 17 Loss 0.6241 Accuracy 0.3328
Time taken for 1 epoch: 87.9146032333374 secs

Epoch 18 Batch 0 Loss 0.5749 Accuracy 0.3618
Epoch 18 Batch 100 Loss 0.5620 Accuracy 0.3440
Epoch 18 Batch 200 Loss 0.5683 Accuracy 0.3421
Epoch 18 Batch 300 Loss 0.5790 Accuracy 0.3422
Epoch 18 Batch 400 Loss 0.5846 Accuracy 0.3403
Epoch 18 Batch 500 Loss 0.5892 Accuracy 0.3391
Epoch 18 Batch 600 Loss 0.5937 Accuracy 0.3382
Epoch 18 Batch 700 Loss 0.5990 Accuracy 0.3372
Epoch 18 Loss 0.5994 Accuracy 0.3372
Time taken for 1 epoch: 87.47308993339539 secs

Epoch 19 Batch 0 Loss 0.5440 Accuracy 0.3380
Epoch 19 Batch 100 Loss 0.5432 Accuracy 0.3466
Epoch 19 Batch 200 Loss 0.5482 Accuracy 0.3428
Epoch 19 Batch 300 Loss 0.5559 Accuracy 0.3430
Epoch 19 Batch 400 Loss 0.5602 Accuracy 0.3428
Epoch 19 Batch 500 Loss 0.5658 Accuracy 0.3420
Epoch 19 Batch 600 Loss 0.5725 Accuracy 0.3412
Epoch 19 Batch 700 Loss 0.5765 Accuracy 0.3405
Epoch 19 Loss 0.5768 Accuracy 0.3405
Time taken for 1 epoch: 86.16471123695374 secs

Epoch 20 Batch 0 Loss 0.5388 Accuracy 0.3664
Epoch 20 Batch 100 Loss 0.5112 Accuracy 0.3502
Epoch 20 Batch 200 Loss 0.5247 Accuracy 0.3485
Epoch 20 Batch 300 Loss 0.5323 Accuracy 0.3481
Epoch 20 Batch 400 Loss 0.5370 Accuracy 0.3466
Epoch 20 Batch 500 Loss 0.5432 Accuracy 0.3459
Epoch 20 Batch 600 Loss 0.5477 Accuracy 0.3442
Epoch 20 Batch 700 Loss 0.5551 Accuracy 0.3436
Saving checkpoint for epoch 20 at /content/drive/My Drive/Colab Notebooks/transformer/ckpt-4
Epoch 20 Loss 0.5552 Accuracy 0.3437
Time taken for 1 epoch: 86.0255196094513 secs

</pre>
</div>
</div>

</div>
</div>

</div>
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
</div><div class="inner_cell">
<div class="text_cell_render border-box-sizing rendered_html">
<h2 id="Evaluate-and-Visualize">Evaluate and Visualize<a class="anchor-link" href="#Evaluate-and-Visualize">&#182;</a></h2><h3 id="Evaluate&#65306;teacher-force">Evaluate&#65306;teacher force<a class="anchor-link" href="#Evaluate&#65306;teacher-force">&#182;</a></h3><p>eg: A B C D -&gt; E F G H</p>
<p>Train: A B C D, E F G -&gt; F G H</p>
<p>Eval:</p>
<ul>
<li>A B C D -&gt; E</li>
<li>A B C D, E -&gt; F</li>
<li>A B C D, E F -&gt; G</li>
<li>A B C D, E F G -&gt; H</li>
</ul>

</div>
</div>
</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="prompt input_prompt">In&nbsp;[&nbsp;]:</div>
<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="k">def</span> <span class="nf">evaluate</span><span class="p">(</span><span class="n">inp_sentence</span><span class="p">):</span>
    <span class="n">start_token</span> <span class="o">=</span> <span class="p">[</span><span class="n">pt_tokenizer</span><span class="o">.</span><span class="n">vocab_size</span><span class="p">]</span>
    <span class="n">end_token</span> <span class="o">=</span> <span class="p">[</span><span class="n">pt_tokenizer</span><span class="o">.</span><span class="n">vocab_size</span> <span class="o">+</span> <span class="mi">1</span><span class="p">]</span>
    <span class="c1"># 文本转id</span>
    <span class="n">input_id_sentence</span> <span class="o">=</span> <span class="n">start_token</span> <span class="o">+</span> <span class="n">pt_tokenizer</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="n">inp_sentence</span><span class="p">)</span> <span class="o">+</span> <span class="n">end_token</span>
    <span class="c1"># 扩维</span>
    <span class="c1"># encoder_input.shape: (1, input_seq_len)</span>
    <span class="n">encoder_input</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">expand_dims</span><span class="p">(</span><span class="n">input_id_sentence</span><span class="p">,</span> <span class="mi">0</span><span class="p">)</span>
    <span class="c1"># decoder_input.shape: (1, 1)</span>
    <span class="n">decoder_input</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">expand_dims</span><span class="p">([</span><span class="n">en_tokenizer</span><span class="o">.</span><span class="n">vocab_size</span><span class="p">],</span> <span class="mi">0</span><span class="p">)</span>
    
    <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">max_length</span><span class="p">):</span>
        <span class="c1"># 创建mask</span>
        <span class="n">encoder_padding_mask</span><span class="p">,</span> <span class="n">decoder_mask</span><span class="p">,</span> <span class="n">encoder_decoder_padding_mask</span> <span class="o">=</span> <span class="n">create_masks</span><span class="p">(</span>
            <span class="n">encoder_input</span><span class="p">,</span> <span class="n">decoder_input</span><span class="p">)</span>
        
        <span class="c1"># 预测</span>
        <span class="c1"># predictions.shape: (batch_size, output_target_len, target_vocab_size)</span>
        <span class="n">predictions</span><span class="p">,</span> <span class="n">attention_weights</span> <span class="o">=</span> <span class="n">transformer</span><span class="p">(</span>
            <span class="n">encoder_input</span><span class="p">,</span> <span class="n">decoder_input</span><span class="p">,</span> <span class="kc">False</span><span class="p">,</span> <span class="n">encoder_padding_mask</span><span class="p">,</span>
            <span class="n">decoder_mask</span><span class="p">,</span> <span class="n">encoder_decoder_padding_mask</span><span class="p">)</span>
        <span class="c1"># 取出预测序列的最后一个</span>
        <span class="c1"># predictions.shape: (batch_size, target_vocab_size)</span>
        <span class="n">predictions</span> <span class="o">=</span> <span class="n">predictions</span><span class="p">[:,</span> <span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="p">:]</span>
        <span class="c1"># 取最大值</span>
        <span class="n">predicted_id</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">cast</span><span class="p">(</span><span class="n">tf</span><span class="o">.</span><span class="n">argmax</span><span class="p">(</span><span class="n">predictions</span><span class="p">,</span> <span class="n">axis</span><span class="o">=-</span><span class="mi">1</span><span class="p">),</span> <span class="n">tf</span><span class="o">.</span><span class="n">int32</span><span class="p">)</span>
        
        <span class="c1"># 判断是否是最后一位</span>
        <span class="k">if</span> <span class="n">tf</span><span class="o">.</span><span class="n">equal</span><span class="p">(</span><span class="n">predicted_id</span><span class="p">,</span> <span class="n">en_tokenizer</span><span class="o">.</span><span class="n">vocab_size</span> <span class="o">+</span> <span class="mi">1</span><span class="p">):</span>
            <span class="c1"># 因为之前扩维，故把第0个维度缩减</span>
            <span class="k">return</span> <span class="n">tf</span><span class="o">.</span><span class="n">squeeze</span><span class="p">(</span><span class="n">decoder_input</span><span class="p">,</span> <span class="n">axis</span> <span class="o">=</span> <span class="mi">0</span><span class="p">),</span> <span class="n">attention_weights</span>
        
        <span class="n">decoder_input</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">concat</span><span class="p">([</span><span class="n">decoder_input</span><span class="p">,</span> <span class="p">[</span><span class="n">predicted_id</span><span class="p">]],</span> <span class="n">axis</span> <span class="o">=</span> <span class="o">-</span><span class="mi">1</span><span class="p">)</span>
    <span class="k">return</span> <span class="n">tf</span><span class="o">.</span><span class="n">squeeze</span><span class="p">(</span><span class="n">decoder_input</span><span class="p">,</span> <span class="n">axis</span> <span class="o">=</span> <span class="mi">0</span><span class="p">),</span> <span class="n">attention_weights</span>
</pre></div>

    </div>
</div>
</div>

</div>
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
</div><div class="inner_cell">
<div class="text_cell_render border-box-sizing rendered_html">
<h3 id="Visualize">Visualize<a class="anchor-link" href="#Visualize">&#182;</a></h3><p>可视化Encoder和Decoder之间的attention_weights</p>

</div>
</div>
</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="prompt input_prompt">In&nbsp;[&nbsp;]:</div>
<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="k">def</span> <span class="nf">plot_encoder_decoder_attention</span><span class="p">(</span><span class="n">attention</span><span class="p">,</span> <span class="n">input_sentence</span><span class="p">,</span> <span class="n">result</span><span class="p">,</span> <span class="n">layer_name</span><span class="p">):</span>
    <span class="n">fig</span> <span class="o">=</span> <span class="n">plt</span><span class="o">.</span><span class="n">figure</span><span class="p">(</span><span class="n">figsize</span><span class="o">=</span><span class="p">(</span><span class="mi">16</span><span class="p">,</span> <span class="mi">8</span><span class="p">))</span>
    <span class="n">input_id_sentence</span> <span class="o">=</span> <span class="n">pt_tokenizer</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="n">input_sentence</span><span class="p">)</span>
    
    <span class="c1"># attention[layer_name].shape: (batc_size=1, num_heads, tar_len, input_len)</span>
    <span class="c1"># attention.shape: (num_heads, tar_len, input_len)</span>
    <span class="n">attention</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">squeeze</span><span class="p">(</span><span class="n">attention</span><span class="p">[</span><span class="n">layer_name</span><span class="p">],</span> <span class="n">axis</span> <span class="o">=</span> <span class="mi">0</span><span class="p">)</span>
    
    <span class="c1"># 画num_heads个子图</span>
    <span class="k">for</span> <span class="n">head</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">attention</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]):</span>
        <span class="n">ax</span> <span class="o">=</span> <span class="n">fig</span><span class="o">.</span><span class="n">add_subplot</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="n">head</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)</span>
        <span class="c1"># 画矩阵,去掉最后一个</span>
        <span class="n">ax</span><span class="o">.</span><span class="n">matshow</span><span class="p">(</span><span class="n">attention</span><span class="p">[</span><span class="n">head</span><span class="p">][:</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="p">:])</span>
        <span class="c1"># 设置字体</span>
        <span class="n">fontdict</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;fontsize&#39;</span><span class="p">:</span> <span class="mi">10</span><span class="p">}</span>
        
        <span class="c1"># 设置锚点</span>
        <span class="n">ax</span><span class="o">.</span><span class="n">set_xticks</span><span class="p">(</span><span class="nb">range</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">input_id_sentence</span><span class="p">)</span> <span class="o">+</span> <span class="mi">2</span><span class="p">))</span>
        <span class="n">ax</span><span class="o">.</span><span class="n">set_yticks</span><span class="p">(</span><span class="nb">range</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">result</span><span class="p">)))</span>
        <span class="n">ax</span><span class="o">.</span><span class="n">set_ylim</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">result</span><span class="p">)</span> <span class="o">-</span> <span class="mf">1.5</span><span class="p">,</span> <span class="o">-</span><span class="mf">0.5</span><span class="p">)</span>
        
        <span class="c1"># 设置label</span>
        <span class="n">ax</span><span class="o">.</span><span class="n">set_xticklabels</span><span class="p">(</span>
            <span class="p">[</span><span class="s1">&#39;&lt;start&gt;&#39;</span><span class="p">]</span> <span class="o">+</span> <span class="p">[</span><span class="n">pt_tokenizer</span><span class="o">.</span><span class="n">decode</span><span class="p">([</span><span class="n">i</span><span class="p">])</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="n">input_id_sentence</span><span class="p">]</span> <span class="o">+</span> <span class="p">[</span><span class="s1">&#39;&lt;end&gt;&#39;</span><span class="p">],</span>
            <span class="n">fontdict</span> <span class="o">=</span> <span class="n">fontdict</span><span class="p">,</span> <span class="n">rotation</span> <span class="o">=</span> <span class="mi">90</span><span class="p">)</span>
        <span class="n">ax</span><span class="o">.</span><span class="n">set_yticklabels</span><span class="p">(</span>
            <span class="p">[</span><span class="n">en_tokenizer</span><span class="o">.</span><span class="n">decode</span><span class="p">([</span><span class="n">i</span><span class="p">])</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="n">result</span> <span class="k">if</span> <span class="n">i</span> <span class="o">&lt;</span> <span class="n">en_tokenizer</span><span class="o">.</span><span class="n">vocab_size</span><span class="p">],</span>
            <span class="n">fontdict</span> <span class="o">=</span> <span class="n">fontdict</span><span class="p">)</span>
        <span class="n">ax</span><span class="o">.</span><span class="n">set_xlabel</span><span class="p">(</span><span class="s1">&#39;Head </span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">head</span> <span class="o">+</span> <span class="mi">1</span><span class="p">))</span>
        
    <span class="c1"># 自适应调整子图位置、间距</span>
    <span class="n">plt</span><span class="o">.</span><span class="n">tight_layout</span><span class="p">()</span>
    <span class="n">plt</span><span class="o">.</span><span class="n">show</span><span class="p">()</span>
</pre></div>

    </div>
</div>
</div>

</div>
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
</div><div class="inner_cell">
<div class="text_cell_render border-box-sizing rendered_html">
<h3 id="Inference:Translate">Inference:Translate<a class="anchor-link" href="#Inference:Translate">&#182;</a></h3>
</div>
</div>
</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="prompt input_prompt">In&nbsp;[&nbsp;]:</div>
<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="k">def</span> <span class="nf">translate</span><span class="p">(</span><span class="n">input_sentence</span><span class="p">,</span> <span class="n">layer_name</span> <span class="o">=</span> <span class="s1">&#39;&#39;</span><span class="p">):</span>
    <span class="n">result</span><span class="p">,</span> <span class="n">attention_weights</span> <span class="o">=</span> <span class="n">evaluate</span><span class="p">(</span><span class="n">input_sentence</span><span class="p">)</span>
    <span class="n">predicted_sentence</span> <span class="o">=</span> <span class="n">en_tokenizer</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span>
        <span class="p">[</span><span class="n">i</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="n">result</span> <span class="k">if</span> <span class="n">i</span> <span class="o">&lt;</span> <span class="n">en_tokenizer</span><span class="o">.</span><span class="n">vocab_size</span><span class="p">])</span>
    
    <span class="nb">print</span><span class="p">(</span><span class="s1">&#39;Input: </span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">input_sentence</span><span class="p">))</span>
    <span class="nb">print</span><span class="p">(</span><span class="s1">&#39;Predicted translation: </span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">predicted_sentence</span><span class="p">))</span>
    
    <span class="k">if</span> <span class="n">layer_name</span><span class="p">:</span>
        <span class="n">plot_encoder_decoder_attention</span><span class="p">(</span><span class="n">attention_weights</span><span class="p">,</span> <span class="n">input_sentence</span><span class="p">,</span> <span class="n">result</span><span class="p">,</span> <span class="n">layer_name</span><span class="p">)</span>
</pre></div>

    </div>
</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="prompt input_prompt">In&nbsp;[72]:</div>
<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="n">translate</span><span class="p">(</span><span class="s1">&#39;está muito frio aqui&#39;</span><span class="p">)</span>
</pre></div>

    </div>
</div>
</div>

<div class="output_wrapper">
<div class="output">


<div class="output_area">

    <div class="prompt"></div>


<div class="output_subarea output_stream output_stdout output_text">
<pre>Input: está muito frio aqui
Predicted translation: it &#39;s very cold , it &#39;s very cold .
</pre>
</div>
</div>

</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="prompt input_prompt">In&nbsp;[73]:</div>
<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="n">translate</span><span class="p">(</span><span class="s1">&#39;istá muito frio aqui&#39;</span><span class="p">)</span>
</pre></div>

    </div>
</div>
</div>

<div class="output_wrapper">
<div class="output">


<div class="output_area">

    <div class="prompt"></div>


<div class="output_subarea output_stream output_stdout output_text">
<pre>Input: istá muito frio aqui
Predicted translation: it goes a lot of cold to this point here .
</pre>
</div>
</div>

</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="prompt input_prompt">In&nbsp;[74]:</div>
<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="n">translate</span><span class="p">(</span><span class="s1">&#39;você ainda está em casa?&#39;</span><span class="p">)</span>
</pre></div>

    </div>
</div>
</div>

<div class="output_wrapper">
<div class="output">


<div class="output_area">

    <div class="prompt"></div>


<div class="output_subarea output_stream output_stdout output_text">
<pre>Input: você ainda está em casa?
Predicted translation: are you still at home ?
</pre>
</div>
</div>

</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="prompt input_prompt">In&nbsp;[75]:</div>
<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="n">translate</span><span class="p">(</span><span class="s2">&quot;este é o primeiro livro que eu fiz.&quot;</span><span class="p">)</span>
</pre></div>

    </div>
</div>
</div>

<div class="output_wrapper">
<div class="output">


<div class="output_area">

    <div class="prompt"></div>


<div class="output_subarea output_stream output_stdout output_text">
<pre>Input: este é o primeiro livro que eu fiz.
Predicted translation: this is the first book i did .
</pre>
</div>
</div>

</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="prompt input_prompt">In&nbsp;[76]:</div>
<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="n">translate</span><span class="p">(</span><span class="s2">&quot;este é o primeiro livro que eu fiz.&quot;</span><span class="p">,</span> <span class="n">layer_name</span><span class="o">=</span><span class="s1">&#39;decoder_layer4_att2&#39;</span><span class="p">)</span>
</pre></div>

    </div>
</div>
</div>

<div class="output_wrapper">
<div class="output">


<div class="output_area">

    <div class="prompt"></div>


<div class="output_subarea output_stream output_stdout output_text">
<pre>Input: este é o primeiro livro que eu fiz.
Predicted translation: this is the first book i did .
</pre>
</div>
</div>

<div class="output_area">

    <div class="prompt"></div>




<div class="output_png output_subarea ">
<img src="
AAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0
dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nOzde5RkdXnv//czd2BmuCOK4aqCoiA4
R0XRg1eMMRe8HCOu/OItGI2XaDgeSYzGRHNMPDErmigiCBr9mRCVE4xBE4wCAmK4zoAwxJ+XeMEk
Aso4MLee5/dH7Z6uaXqG6q79rd3f6vdrrVrdtbv62U/vrvrM7mf23hWZiSRJkiRJkuq1qOsGJEmS
JEmSNBwHPJIkSZIkSZVzwCNJkiRJklQ5BzySJEmSJEmVc8AjSZIkSZJUOQc8kiRJkiRJlXPAI0mS
JEmSVDkHPJIkSZIkSZVzwCNJkiRJklQ5BzySJEmSJEmVW9J1A5r/IuJ44CnN3Ssy86Yu+5E03swc
SaNk5kgaNXNHpXgEj3YrIt4IfBI4qLl9IiJe321XksaVmSNplMwcSaNm7qikyMyue9A8FhFrgZMy
c2Nzfy/g6sw8rtvOJI0jM0fSKJk5kkbN3FFJHsGjBxLARN/9iWaZJJVg5kgaJTNH0qiZOypmbK7B
ExEBXASclZm3dt3PGDkfuCYiLmru/wpwXof9SPOGuVOEmSPtgplThJkj7YKZU4y5o2LG5hStiDgV
+CjwN5n5O133M04i4kTg5ObuFZl5Q5f9SPOFuVOGmSPNzMwpw8yRZmbmlGPuqJRxGvBcSG8a+hfA
ozJzW8ctVS8iFgO3ZOYxXfcizUfmTrvMHGn3zJx2mTnS7pk57TN3VNpYXIMnIg4Ajs3MS4BL6R3m
piFl5gSwPiIO7boXab4xd9pn5ki7Zua0z8yRds3MKcPcUWljMeABfg34VPP5+cCrOuxl3OwL3BIR
X4qIiydvXTelbkXEaRGxsus+OmbulGHm6H7MHMDMKcXM0f2YOYCZU5K5o/tpK3fG4hStiFgHPCcz
f9Dcvwl4XmZ+r9vO6hcR/32m5Zl52ah70fwQEUcBtwGvz8yzu+6nK+ZOGWaOpjNzesycMswcTWfm
9Jg55Zg7mq7N3Kl+wBMR+wAvzswP9y17FvBjL1YltS8i3tV8+uzMfHynzXTE3JFGx8wxc6RRMnPM
HGnU2syd6k/RysyfADdPW/bPwJ7ddDQeIuKrzccNEXFP321DRNzTdX/qRnNhuBcBfwL8NCKO77il
Tpg77TNzNBMzp8fMaZ+Zo5mYOT1mThnmjmbSdu5UP+BpfGDAZRpQZp7cfFyVmav7bqsyc3XX/akz
zwW+lpkb6L1t5is77qdL5k6LzBztgpkzxcxpkZmjXTBzppg5LTN3tAut5s6SVlrqSEScBDwJODAi
3tz3pdXA4m66Gj8RcTLw8Mw8v7mi/qrM/HbXfakTrwTe13x+EfCuiDgzM7d02NNImTvlmTnqY+aY
OcWZOepj5pg5I2HuqE+ruVP7ETzLgJX0BlWr+m73AC/ssK+xERHvAP4XcFazaBnwie46Ulea87H3
yczLATJzE/Bp4OmdNjZ65k5BZo4mmTk7mDkFmTmaZObsYOYUZu5oUoncGYeLLC8GLszMF3TdyziK
iBuBE4DrM/OEZtnazDyu286k7pg75Zg50v2ZOeWYOdL9mTllmTsqqepTtAAycyIiHtJ1H2NsS2Zm
RCRAROzVdUMavYg4cXdfz8zrR9XLfGDuFGXmyMyZxswpysyRmTONmVOcuaNiuVP9gKdxY0RcDPwd
sHFyYWZ+truWxsaFEfFhYJ+I+A3gFcBHOu5Jo/dnzccVwBrgJiCA44BrgZM66qtL5k4ZZo7AzJmJ
mVOGmSMwc2Zi5pRj7ggK5U71p2gBRMT5MyzOzHzFyJsZQxHxLODZ9J5wX2zeJlELUER8FnhHZq5r
7j8a+IPMXHDnZJs75Zg5mmTmTDFzyjFzNMnMmWLmlGXuaFLbuTMWAx6VFxGr6TviKzPv6rAddSQi
bsnMYx9omTQsM0dg5mh0zByBmaPRMncE7efOWJyiFREr6L292LH0DnECwAnz8CLi1cA7gU3AdnpT
5gSObKH2MuARzd31mbl12Joqbm1EnMvUlf5fCqztsJ/OmDtlmDmaxsxpmDllmDmaxsxpmDnllMod
M6dareZO7W+TPumvgYOBU4HLgIcCG4YpGBEPiojzIuKS5v6jIuKVQ3danzOBR2fm4Zl5ZGYekZlt
7PScAvwb8FfAB4HbI+Kpw9ZVcS8HbgHe2Ny+0SxbiFrNHTNnBzNH/cycKe7rlGHmqJ+ZM8XMKaf1
3DFzqtZq7ozFKVoRcUNmnjD59nIRsRS4IjOfOETNS4Dzgd/LzOMjYglwQ2Y+pq2+axARXwCen5n3
tlz3OuD0zFzf3H8E8KnMfFyb65FKaTt3zJweM0eamfs6ZZg50szMnHJK5I6Zo0ljcYoWMHn42U+a
ixL9CDhoyJoHZOaFEXEWQGZui4iJIWvW6Czgqoi4Btg8uTAz3zBk3aWTAdTUu735h2MoEXEY8PDM
vDQi9gCWZOZQ/9ugKRHxZOAPgMPY+Zzhof+3s0Jt546Z01NV5oC5U5KZsxP3dcowc7SDmbMTM6ec
Erlj5lSq7dwZlwHPORGxL/A24GJgJfD7Q9bcGBH70zsfkoh4IvDTIWvW6MPAvwDr6J0j2pZrZzjX
8NphCkbvbQbPAPYDjqJ3KOnZwDOGqaudnAe8CbgOWIj/IPdrO3fMnJ5qMgfMnREwc6a4r1OGmaN+
Zs4UM6ecErlj5tSr1dwZl1O0jsjMbz/QslnWPBH4APBo4GbgQOBFmXnTUM1WZvLwzAJ1lwO/BZzc
LLoC+GBmbt71dz1gzRuBxwPXTPYcEesW2mGfJUXENZn5hK77mA/azh0zp6emzGnqmjsFmTlT3Ncp
w8xRPzNniplTToncMXPq1XbujMuA5/rMPHHasuuGOeeweZFMAEfTu7L5emDRsC+S2kTEHwPfAT7H
zocQzvlt/CJiMfDxzHzp0A3uXPeazHxC3znDS4DrM/O4NtezkEXEe4DFwGfZ+flwfWdNdaTt3DFz
emrKnKa2uVOQmTPFfZ0yzBz1M3OmmDnltJ07Zk7d2s6dqk/Riohj6L11394R8fy+L62m7+385ujq
JtRu6Vvf9cCJu/6WsfSS5uNZfcuGehu/zJyIiMMiYllmbhmqu51dFhG/C+wREc8CXksvONWeyeny
mr5lCTy9g146UTB3zJyemjIHzJ3SzBz3dUozc9TPzDFzRqHV3DFzqtdq7lQ94KE3/X0esA/wi33L
NwC/MZeCEXEwcAi9J/EJ9KbL0Au1Pefeap0y84hCpb8FXBkRFwMb+9b3viFqvhV4Jb3zWV8N/CNw
7jBNameZ+bSue5gHWs0dM2dnlWUOmDtFmTmA+zpFmTnqZ+YAZk5xhXLHzKlU27kzLqdonZSZV7dU
69eBl9GboP0rUwG0AbggMz/bxnrmu4h4emb+y7TJ/Q7DboeIeMcu6r5zmLoqKyIeBPwx8JDM/PmI
eBRwUmae13FrI9dW7pg5PWaOZmLmTHFfp11mjmZi5kwxc9pXMnfMnHq1nTvjMuD5U+BdwH3AF4Dj
gDdl5id2+427r/mCzPxMSy1WJyLemZnviIjzZ/hyZuYrhqx/YtvnM0fEt2muyt8vh3xry2YbzFR3
qG1Qo4i4BDgf+L3MPL45D/eGhXihtbZzx8ypL3OauuZOQWbOFPd12mXm3K+umYOZ08/MaV/J3DFz
6tV27tR+itakZ2fmWyLiNHoXrHo+cDlTbxM3Fw+NiNX0JssfoXdu6Fsz85+GbbYGTfgsAi7JzAsL
rOLPmsM1Pw38bWbe3ELN/vMWVwAvoveWfsP6h2l1TwN+OGzRiHgKcFVmTvQtKxLOLTogMy+MiLMA
MnNbRCzUtxFtO3fMnPoyB8yd0sycKe7rtMjMuR8zp8fMmWLmtKxw7pg5VJk50HbuZGb1N+CW5uO5
wHOaz28asuZNzcdTgYvoXWzs+q5/1g627bUFax8MvAG4kt55nW8rsI7rCtRcRC84hq1zL3AZcFDf
snn9HAO+Auw/2SfwROCyrvvqaFu0mjtmzo7tUHXmNOsxd9r7uc2cqW3hvk6Z7WrmzFzTzEkzp/lo
5rS/bYvkjplTX+Y0/bWaO+NyBM/nIuI2eocQviYiDgQ2DVlz8tzQ59J727lbIiJ29w1j6tKIOBP4
W3a+YNec3z60r8aPgPdHxJeBtwBvp3co6JxERP8V+BfRmziXeI4/HDiohTrrgffSuzr9KzPzKqae
d/PVm4GLgaMi4krgQOCF3bbUmbZzx8zpqSZzwNwZATNnivs6ZZg5MzNzzBwzp5wiuWPmAPVlDrSc
O2NxDR6AiNgP+Gn23iZuL2BV8ySfa73z6V3t/QjgeHrvTf+VzHxcKw1XojnvcrrM4c+7fCTwYuAF
wJ30Au4zmfmfQ9T8ct/dbfQOJ/0/mbl+iFaJiA30zhGN5uOPgLNyyHOII+L6zDwxIh5O7+f/KPCK
7L195LzVnBd6NL3tsT4zt3bcUmfazB0zp6emzGnqmjuFmTlT3Ndpn5mzo66Z0zBzppg5ZZTIHTNn
R93qMgfazZ3qBzwRsSfw8My8qW/ZocBEZv5giLqLgMcC38rMn0TE/sAhmbl26KZFRFxN70V3YWYO
fb5ljSLihsw8ofl8Jb0Aen5mzssj60q91mpUYluYOWWZOT015Y6ZM8V9nfqYOT1mTp3MnPqYOT01
ZQ4U+ptiDAY8S4HbgOMyc2Oz7J+A383Ma4eoG8BLgSMz8w+bDX1wZn59yH6L1G1qHw88pbl7Rf8T
ZYiaK4DXAifTm65eAZydmcMeotm6iHjz7r6eme+bY93J39kRmflHbf7OZljXoZn5723XbUOp11qN
SmwLM2dHzWoyB8ydksycKe7r7Khr5pg5xZg5U8ycnWov6Nwxc8oq8Vpb1GJ/nWgOX7oI+B+wY+J1
YAtB/EHgJOAlzf0NwF/NpVBEnBwRi9uuO20dbwQ+Se/cxYOAT0TE64etC3yc3gXQPgD8ZfP5Xw/R
54XNx3URsbbvti4ihp3erwFeQ+/Qz0OA36R3df5VzW2uJn9npzf3h/qdRcRbmo8fiIj399+AM4fo
s6iCr7XqFNoWZk5PTZkD5k4xZs4U93XMnD5mTiFmzhQzZ8c65n3umDk9NWYOFHqt5Ty4cvSwN+AY
4PLm87cBb2ih5uRVrG/oWzanK8cDTwLOabvutHWsBfbqu78XsLaFut8YZNks6j24+XjYTLche72c
3rnBk/dXTT4v5stzofneO5uPvw38+vTbsP2WvJV4rdV6a3tbmDk76lSTOU1dc6fgzcwpuy1qyh0z
Z0d9M6fgzcwpuy1qypymxrzPHTNnx/dWmTlNz62+1ubluWizlZm3Rc8jgF9l6jC6YWxtpsIJEL0r
x2+fY39XRcS9bdedJoCJvvsTzbJhXR8RT8zMrwFExBOAOU8UM/OO5uN3W+htugcBW/rub2mWDavt
39l/RMRDgJcDp9DO72kkCr3WqlRgW5g5PTVlDpg7RZk5U9zXMXMaZk5BZs4UMweoIHfMnB2qzBxo
/7U2FgOexnnAucC6zLy7hXrvp3e41EER8W56b1X2trkWy8wbS9Ttcz5wTURc1Nz/FXrbZFiPA66K
iMnzFg8F1kfEOnpXez9uNsVi6orp9/tSU2/1EL1+HPj6tG1wwRD1JrX9O/sQ8CXgSOC6vuWTV5Ef
6p07diUiDs4h3vmgT9uvtZq1uS3MnJ6aMgfMnV0yc4pYyPs6Zk6PmbMbLeWOmTNlIWcOVJA7Zs4O
NWcOtPhaq/4iy5OidwXqO4AXZOalLdU8BngGvSfGlzLz1nle90R6F+uC3kXAbmih5mG7+3rBafGc
NNtgcup5eRvboKnb+u8sIj6Uma8ZurnB1/f5zPyFFuq0/lqrVdvbwsypL3PA3NnNusycli30fR0z
p8fM2e36hs4dM2fKQs+cpu6Czx0zZ7frm3f7OmMz4JEkSZIkSVqoqn8XLUmSJEmSpIXOAY8kSZIk
SVLlxm7AExFnWNe6pWpat3zdGvn8tW6NdWvqtWTdGtW2jX3+WrfGumbOlJp+b9YtV9O6ddQduwEP
UCqMrVtf3Zp6tW7dfP5at8a6NfVasm6NatvGPn+tW2NdM2dKTb8365arad0K6o7jgEeSJEmSJGlB
qeJdtJbFitwj9hrosVvYzDKWD/TYTQ/dc+AeJn62kcUrB+thxY82D1x3y/b7WLZoj4Eem9smBq67
NTexNFYM9NiJ/QbfDts2bWTJigfeDkvuvm/gmgBbchPLBug3t28fuOZWNrN0wOfCbFh39nU3cPeP
M/PA1psoaNmiFbnH4lUDPXbL9k0sWzTY623/Y+4d6HEb7trGqv2WDPRYgDtvHex1POhrbVJODJY7
8+F5Zt3ua86XupvYyJbcHK03UdCypXvlihX7DPTYrVs3snTpYPsksWnLwD3MJsuYxe7jlryPZTHY
vg4x2K9tNvtPAJsfNNhzZ2LjRhbvNdi2XX7X4PtlW7ZtZNmSwermpk0D192am1kag77eBn9JzGY/
kln8LTEf8mHQ59hstsGm3MiW3FRX5sTyXMFgz8nZbN8YcPvC7P5uO+jYwfafAH561wR777d4oMf+
x82D58istsPyZQPX3TJxL8sWP/B+XG4ePM/nxWvNukXr7urvq8H/eujQHrEXT1zx3Nbrrv+fj229
JsAj3/OdInUn7rq7SN27funE1mvu/5mbW68JsP1nPytSdzY7J7Myi3/kxtml2//uu133MFt7LF7F
Sfu+oPW6L7vo2tZrAlzwhPZfxwATP72nSF1y8GHt4DXn/39YaAYFcvKa7Ze2XrO0FSv24fGPfW3r
dZfc9u+t1wRgwOHvrC0d/I+i2fjm6x/Wes2jPlVmvyy/8c0idVk82B+8s5VbBv+jcz6IZe0/x762
+ZLWa5a2gr14Qjyj9bqLVgz+n0iz8ca/v6FI3T9/2COL1F3y0MNbr7nt24V2p91/qtKl+ekZnxCe
oiVJkiRJklQ5BzySJEmSJEmVc8AjSZIkSZJUOQc8kiRJkiRJlXPAI0mSJEmSVLk5DXgiYp+IeG3f
/VMi4h928dhzI+JRc21QkswcSaNk5kgaNXNHUhvmegTPPsBA7+WZma/KzG/McT2SBGaOpNEycySN
mrkjaWhzHfC8BzgqIm6MiPc2y1ZGxKcj4raI+GREBEBEfCUi1kTE4oi4ICJujoh1EfGmVn4CSQuB
mSNplMwcSaNm7kga2pI5ft9bgUdn5mOhdwghcAJwLPBD4ErgycBX+77nscAhmfno5nv22d0KIuIM
4AyAFbHXHNuUNCaKZ07zmKncWbSyxfYlVWb0mbN87xbbl1Sh0f59xZ4tty9pPmjzIstfz8zvZ+Z2
4Ebg8Glf/xZwZER8ICKeA9yzu2KZeU5mrsnMNctY3mKbksZEq5kD03Jn0Yr2O5ZUs6KZs3Sp/5kl
6X6K/X211L+vpLHU5oBnc9/nE0w7Oigz7waOB74C/CZwbovrlrTwmDmSRsnMkTRq5o6kWZnrKVob
gFWz+YaIOADYkpmfiYj1wCfmuG5JC4+ZI2mUzBxJo2buSBranAY8mXlnRFwZETcDlwCfH+DbDgHO
j4jJo4bOmsu6JS08Zo6kUTJzJI2auSOpDXM9gofMPH3aoq/0fe11fZ+f0veYE+e6PkkLm5kjaZTM
HEmjZu5IGlab1+CRJEmSJElSBxzwSJIkSZIkVc4BjyRJkiRJUuUc8EiSJEmSJFXOAY8kSZIkSVLl
5vwuWiO1aBGxYnnrZY8+7yet1wRg2dIiZRcfsH+Runc+c1PrNQ+4ZI/WawKwYUOZuqVkdt2B5mrR
ImKvPVsv+7HnPK31mgB3nP6QInUf/De3Fal796mPaL3mfld8v/WaABM/+s8idXNiokhdcnuhuoXy
zJwEIDZtZcntP2i97ubHHtF6TYDl3/9pkbrcUeb19nP/vKX1mvcdsqr1mgDL15XJhihSFRY/qv08
B5i49ZtF6uaW9p8L5tiURYX+Xvm5JYX+bitk27e+03ULWqA8gkeSJEmSJKlyDngkSZIkSZIq54BH
kiRJkiSpcg54JEmSJEmSKueAR5IkSZIkqXIOeCRJkiRJkirngEeSJEmSJKlyxQc8EXFV6XVI0iQz
R9KomTuSRsnMkbQrxQc8mfmk0uuQpElmjqRRM3ckjZKZI2lXRnEEz8+ajw+OiMsj4saIuDkinlJ6
3ZIWHjNH0qiZO5JGycyRtCtLRriu04EvZua7I2IxsOfuHhwRZwBnAKxYtNcI2pM0ZmaVOTAtdxav
KtyepDE0xL7OyhG0J2nMzD1zHni3SFKFRjng+VfgoxGxFPi/mXnj7h6cmecA5wDsveTAHEF/ksbL
rDIHpuXO8geZO5Jma+77OksPMnMkzdacM2d17GfmSGNoZO+ilZmXA08FfgBcEBH/z6jWLWnhMXMk
jZq5I2mUzBxJ041swBMRhwH/kZkfAc4FThzVuiUtPGaOpFEzdySNkpkjabpRnqJ1CvA/I2Ir8DPA
CbOkkk7BzJE0Wqdg7kganVMwcyT1KT7gycyVzcePAR8rvT5JC5uZI2nUzB1Jo2TmSNqVkZ2iJUmS
JEmSpDIc8EiSJEmSJFXOAY8kSZIkSVLlHPBIkiRJkiRVzgGPJEmSJElS5Ub5NulzlhMTTPzkp+0X
/uk97dcEcsnSMnUnJorU/YeTP916zTdveGbrNaVRyq1bmfjBHe3X3bat9ZoAB33oe0Xq3v35w4vU
XXr29tZr5n33tV4TILdtLVKXzDJ1VaWcmGD7Pe3vlyz5l+tbrwkwEWX+j3Dx/vsVqfudV7SfOftf
uqz1mgDLC2VDqX9/Jm5ZX6Su6rTt+z8oUvfMY55WpC6Lyvwb/85vfr31mn/05F9ovSbAtjt+VKSu
uuERPJIkSZIkSZVzwCNJkiRJklQ5BzySJEmSJEmVc8AjSZIkSZJUOQc8kiRJkiRJlXPAI0mSJEmS
VDkHPJIkSZIkSZUbasATEftExGv77p8SEf8wfFuSdH9mjqRRMnMkjZq5I2kYwx7Bsw/w2gd8lCS1
w8yRNEpmjqRRM3ckzdmwA573AEdFxI0R8d5m2cqI+HRE3BYRn4yIAIiIx0XEZRFxXUR8MSIePOS6
JS08Zo6kUTJzJI2auSNpzpYM+f1vBR6dmY+F3iGEwAnAscAPgSuBJ0fENcAHgF/OzP+KiBcD7wZe
savCEXEGcAbACvYcsk1JY6JY5jT1zB1J/cwcSaPm31eS5mzYAc9Mvp6Z3weIiBuBw4GfAI8G/rkZ
OC8G7thdkcw8BzgHYHXslwX6lDQeWskcmJY7i8wdSTMqlDn7mzmSdsW/ryQNpMSAZ3Pf5xPNOgK4
JTNPKrA+SQubmSNplMwcSaNm7kgayLDX4NkArBrgceuBAyPiJICIWBoRxw65bkkLj5kjaZTMHEmj
Zu5ImrOhBjyZeSdwZUTc3HcRsJketwV4IfAnEXETcCPwpGHWLWnhMXMkjZKZI2nUzB1Jwxj6FK3M
PH3aoq/0fe11fZ/fCDx12PVJWtjMHEmjZOZIGjVzR9JcDXuKliRJkiRJkjrmgEeSJEmSJKlyDngk
SZIkSZIq54BHkiRJkiSpcg54JEmSJEmSKjf0u2hVLbNM2a1bitQt5ZHL9my95vZ77229pjRSCblt
W9ddDG77RJGyq3/+/ytS94s//EzrNU99yGNbrymNTCa5eXPXXQwuy2TOxI9/XKTuN572j63X/OWX
ndR6TYAye6dS3bZv2VqocJkse+KKxa3X3Paj/2i9psaPR/BIkiRJkiRVzgGPJEmSJElS5RzwSJIk
SZIkVc4BjyRJkiRJUuUc8EiSJEmSJFXOAY8kSZIkSVLlHPBIkiRJkiRVbtYDnoh4Q0TcGhGfjIhf
ioi3zuJ7D4+I02e7TkkLm7kjaZTMHEmjZOZIasuSOXzPa4FnZub3m/sXT39ARCzJzG0zfO/hwOnA
/zuH9UpauMwdSaNk5kgaJTNHUitmNeCJiLOBI4FLIuKjwN3Amsx8XURcAGwCTgCujIi/B/6i+dYE
ngq8B3hkRNwIfCwz/7ydH0PSuDJ3JI2SmSNplMwcSW2a1YAnM38zIp4DPC0zfxwRL5v2kIcCT8rM
iYj4HPBbmXllRKykF05vBc7MzOc90Loi4gzgDIAV7DmbNiWNEXNH0iiZOZJGycyR1Ka2L7L8d5k5
0Xx+JfC+iHgDsM8uDincpcw8JzPXZOaapSxvuU1JY8TckTRKZo6kUTJzJA2s7QHPxslPMvM9wKuA
PegdUnhMy+uSJDB3JI2WmSNplMwcSQOby0WWBxIRR2XmOmBdRPw34Bjge8CqUuuUtLCZO5JGycyR
NEpmjqQH0vYRPP1+OyJujoi1wFbgEmAtMBERN0XEmwquW9LCZO5IGiUzR9IomTmSdmvWR/Bk5uF9
n18AXNB8/rJpj3v9Lko8fbbrlLSwmTuSRsnMkTRKZo6ktpQ8gkeSJEmSJEkj4IBHkiRJkiSpcg54
JEmSJEmSKueAR5IkSZIkqXLF3iZ9QYsoU3bJ0iJ1n3vMU1uv+b7vfLH1mgBvPvykInUlzWzxvvsW
qfsLJ57aes3TvnFz6zUB/n7NYUXqbt+0uUhdtk+UqStV7JePPLn1mp/69pdbrwnwq4c+uUhdMsvU
lUZg0R4ritTdfu+9ReqeesgJrdf85L9/tfWaAC/9uUKZo054BI8kSZIkSVLlHPBIkiRJkiRVzgGP
JEmSJElS5RzwSJIkSZIkVc4BjyRJkiRJUuUc8EiSJEmSJFXOAY8kSZIkSVLlHPBIkiRJkiRVbrcD
nog4PCJubmNFEfGdiDigjVqSxpOZI2nUzB1Jo2TmSCrJI3gkSZIkSZIqN8iAZ0lEfDIibo2IT0fE
ngAR8YyIuCEi1kXERyNi+e6WT4qIPSLikoj4jQI/j6T6mTmSRs3ckTRKZo6kIgYZ8BwNfDAzHwnc
A7w2IlYAFwAvzszHAEuA1+xqeV+tlcDngE9l5kd2t9KIOCMiro2Ia7eyeZY/lqSKdZI5YO5IC5j7
OpJGycyRVMQgA57vZeaVzVYlbQwAACAASURBVOefAE6mF0rfzszbm+UfA566m+WT/h44PzM//kAr
zcxzMnNNZq5ZyvIHerik8dFJ5oC5Iy1g7utIGiUzR1IRgwx48gHuz8aVwHMiIoaoIWm8mTmSRs3c
kTRKZo6kIgYZ8BwaESc1n58OfBVYDxweEQ9rlv8acNlulk96O3A38FfDNi5pbJk5kkbN3JE0SmaO
pCIGGfCsB34rIm4F9gU+lJmbgJcDfxcR64DtwNm7Wj6t3huBPSLiT9v6ISSNFTNH0qiZO5JGycyR
VMSS3X0xM78DHLOLr30JOGEWyw/vu/vy2TQpaWEwcySNmrkjaZTMHEklDXIEjyRJkiRJkuYxBzyS
JEmSJEmVc8AjSZIkSZJUOQc8kiRJkiRJldvtRZbnjQhi+fLWy+bmza3XLGnxAft13cLAfu1P3lyk
7pa3RpG6h559S5G62zfeV6Quub1M2W3bitSt1qLF7dfcPtF+zYIm7r676xYG9n+PP6RI3Xff/pUi
dX//+GcWqTtxzz1F6hJl8pfMMnUrE4sXsXjl6tbrTmzY0HpNAKLQ/xE+/tgiZe86eq/Waz7vd+53
zdtW7P3Iu4rUze/+oEzdyvYdcsuWAkXbL1lcBLF0Wetlc2uB7Qtsv29TkbqL99mnSN3Yc4/Wa572
xjJ/X3FambIrflzmubD09jJZFsvafz0AbL+r0L70z2Ze7BE8kiRJkiRJlXPAI0mSJEmSVDkHPJIk
SZIkSZVzwCNJkiRJklQ5BzySJEmSJEmVc8AjSZIkSZJUOQc8kiRJkiRJlXPAI0mSJEmSVLlOBjwR
cVUX65W0cJk7kkbJzJE0SmaOJOhowJOZT+pivZIWLnNH0iiZOZJGycyRBN0dwfOzLtYraeEydySN
kpkjaZTMHEkAS7puYFci4gzgDIAV7NlxN5IWAnNH0ijtlDmxV8fdSBp37udI42/eXmQ5M8/JzDWZ
uWZprOi6HUkLwE65w/Ku25E05vozZ9ki93UkleXfV9L4m7cDHkmSJEmSJA3GAY8kSZIkSVLlHPBI
kiRJkiRVrqu3SV/ZxXolLVzmjqRRMnMkjZKZIwk8gkeSJEmSJKl6DngkSZIkSZIq54BHkiRJkiSp
cg54JEmSJEmSKrek6wYGkzAx0XUTg8ssUnbbHT8qUpeI1kse+OH/ar0mwOJ99y5Sd8sJRxWp+19v
vK9I3UNefWeRutvv+kmRumwpU7a47RXljsitZZ5ob3/KaUXqxt7tZy/A7e89ukjdoz9SJs8W3be1
9Zrxza+2XrO0nNjOxD33dN3G4LJQPn5tbZGy+36tSNkiavuX54s/vLFI3VMPOaFI3VL76dXJLPbv
ZhGF9skm7r67SF0K1N3zsz9svSZQ5G9BgFi8uEjdz33360XqPvehjytSd9SZ4xE8kiRJkiRJlXPA
I0mSJEmSVDkHPJIkSZIkSZVzwCNJkiRJklQ5BzySJEmSJEmVc8AjSZIkSZJUOQc8kiRJkiRJlVtS
snhE/AHwM2A1cHlmXjrt66cAZ2bm80r2IWlhMHMkjZq5I2mUzBxJu1N0wDMpM98+ivVIEpg5kkbP
3JE0SmaOpJm0fopWRPxeRNweEV8Fjm6WXRARL2w+f05E3BYR1wPPb3v9khYWM0fSqJk7kkbJzJE0
qFYHPBHxOOBXgccCzwX+27SvrwA+Avwi8Djg4DbXL2lhMXMkjZq5I2mUzBxJs9H2ETxPAS7KzHsz
8x7g4mlfPwb4dmb+W2Ym8IldFYqIMyLi2oi4dmtubrlNSWOitcyBabmDuSNpRmX2dcwcSTMzcyQN
bN6+i1ZmnpOZazJzzdJY3nU7khaAnXIHc0dSWWaOpFEyc6Tx1/aA53LgVyJij4hYRe9QwX63AYdH
xFHN/Ze0vH5JC4uZI2nUzB1Jo2TmSBpYq++ilZnXR8TfAjcB/wn867Svb4qIM4DPR8S9wBXAqjZ7
kLRwmDmSRs3ckTRKZo6k2Wj9bdIz893Au3fz9S/QO1dUkoZm5kgaNXNH0iiZOZIGNW+vwSNJkiRJ
kqTBOOCRJEmSJEmqnAMeSZIkSZKkyjngkSRJkiRJqpwDHkmSJEmSpMq1/i5aJcTixSzae3XrdSfu
vKv1mlXKLFBzov2aQG7aXKTu8u/cWaTuQ1+3rUjdb7zrsCJ1H/axhxSpyxVlypYUe6xg0SPaf0OK
7Wtva72mytr2/R903cKsPOLV3y9Sd8MXjixSd8937916zVxS3/9fxR4rWHT0I1uvu/2mW1uvWVRE
mbol9nUK9RpLlhapy6Iy/T73+GcVqXv72YcXqXvEZ9p/LuTVV7des7gIYumy1svm1i2t11RZix9x
VJG6E3vvUaTumnc/vkjdlaeV+ft11c0/LlKX9TMvrm8PSJIkSZIkSTtxwCNJkiRJklQ5BzySJEmS
JEmVc8AjSZIkSZJUOQc8kiRJkiRJlXPAI0mSJEmSVDkHPJIkSZIkSZVzwCNJkiRJklQ5BzySJEmS
JEmVc8AjSZIkSZJUuSVdN7ArEXEGcAbAikUrO+5G0kKwU+4sXd1xN5LG3c6Zs3fH3UgadztlDnt2
3I2kEubtETyZeU5mrsnMNcsWrei6HUkLwE65s2SvrtuRNOZ2zhz/2JJUVn/mLA3/vpLG0bwd8EiS
JEmSJGkwDngkSZIkSZIq1/mAJyLOjYg1XfchaWEwcySNmrkjaZTMHGnh6vwiy5n5qq57kLRwmDmS
Rs3ckTRKZo60cHV+BI8kSZIkSZKG44BHkiRJkiSpcg54JEmSJEmSKueAR5IkSZIkqXIOeCRJkiRJ
kioXmdl1Dw8oIv4L+O6ADz8A+HGBNqxbX92aeh33uodl5oEFeihmHuTOfPi9WXe869bU62zrmjlz
U1Pdmnq1bp11zZwpNf3erFuupnXnV90Zc6eKAc9sRMS1mbnGutatqVfr1s3nr3VrrFtTryXr1qi2
bezz17o11jVzptT0e7NuuZrWraOup2hJkiRJkiRVzgGPJEmSJElS5cZxwHOOda1bsKZ1y9etkc9f
69ZYt6ZeS9atUW3b2OevdWusa+ZMqen3Zt1yNa1bQd2xuwaPRi8ifpaZK/vuvwxYk5mva6H2V4Az
M/PaactfB/w2cBRwYGaWuMiVpHmoo8z5JLAG2Ap8HXh1Zm4ddn2S5r+OMuc8epkTwO3AyzLzZ8Ou
T1Idusidvq+/H3hF//pVj3E8gkcLw5XAMxn86v+SNIxPAscAjwH2AF7VbTuSxtybMvP4zDwO+Hdg
6D/qJOmBRMQaYN+u+9DcOeBRURFxYER8JiL+tbk9uVn++Ii4OiJuiIirIuLoZvkeEfE3EXFrRFxE
7w+p+8nMGzLzO6P7SSTVoGDm/GM26B3B89CR/VCS5q2CmXNP8/hoHuMh95KAcrkTEYuB9wJvGdkP
o9Yt6boBjYU9IuLGvvv7ARc3n/8F8OeZ+dWIOBT4IvBI4DbgKZm5LSKeCfwx8ALgNcC9mfnIiDgO
uH5kP4WkWnSWORGxFPg14I2t/kSS5rNOMicizgeeC3wD+J22fyhJ81oXufM64OLMvKM3W1aNHPCo
Dfdl5mMn70yeI9rcfSbwqL6QWB0RK4G9gY9FxMPp/a/U0ubrTwXeD5CZayNibfn2JVWmy8z5IHB5
Zl7Rxg8iqQqdZE5mvrz5H/UPAC8Gzm/tJ5I03400dyLiIcCLgFNa/0k0Ug54VNoi4ImZual/YUT8
JfDlzDwtIg4HvjL61iSNoWKZExHvAA4EXj18m5LGRNH9nMyciIi/oXfKhAMeSVAmd04AHgZ8sxkc
7RkR38zMh7XSsUbGa/CotH8CXj95JyImJ9F7Az9oPn9Z3+MvB05vHvto4LjyLUoaI0UyJyJeBZwK
vCQzt7fbsqSKtZ450fOwyc+BX6J36oUkQYHcyczPZ+bBmXl4Zh5O75QuhzsVcsCj0t4ArImItRHx
DeA3m+V/CvzviLiBnY8k+xCwMiJuBf4QuG6mohHxhoj4Pr0Lna6NiHOL/QSSalIkc4CzgQcBV0fE
jRHx9jLtS6pMicwJeqdZrAPWAQ9uHitJUG5fR2Mgem8IIkmSJEmSpFp5BI8kSZIkSVLlHPBIkiRJ
kiRVzgGPJEmSJElS5RzwSJIkSZIkVc4BjyRJkiRJUuUc8EiSJEmSJFXOAY8kSZIkSVLlHPBIkiRJ
kiRVzgGPJEmSJElS5RzwSJIkSZIkVc4BjyRJkiRJUuUc8EiSJEmSJFXOAY8kSZIkSVLlHPBIkiRJ
kiRVzgGPJEmSJElS5RzwSJIkSZIkVc4BjyRJkiRJUuUc8EiSJEmSJFXOAY8kSZIkSVLlHPBIkiRJ
kiRVzgGPJEmSJElS5RzwSJIkSZIkVc4BjyRJkiRJUuUc8EiSJEmSJFXOAY8kSZIkSVLlHPBIkiRJ
kiRVzgGPJEmSJElS5RzwSJIkSZIkVc4BjyRJkiRJUuUc8EiSJEmSJFXOAY8kSZIkSVLlHPBIkiRJ
kiRVzgGPJEmSJElS5RzwSJIkSZIkVc4BjyRJkiRJUuUc8EiSJEmSJFXOAY8kSZIkSVLlHPBIkiRJ
kiRVzgGPJEmSJElS5RzwSJIkSZIkVW5J1w1o/ouI44GnNHevyMybuuxH0ngzcySNkpkjadTMHZXi
ETzarYh4I/BJ4KDm9omIeH23XUkaV2aOpFEycySNmrmjkiIzu+5B81hErAVOysyNzf29gKsz87hu
O5M0jswcSaNk5kgaNXNHJXkEjx5IABN99yeaZZJUgpkjaZTMHEmjZu6omLG5Bk9EBHARcFZm3tp1
P2PkfOCaiLiouf8rwHkd9iPNG+ZOEWaOtAtmThFmjrQLZk4x5o6KGZtTtCLiVOCjwN9k5u903c84
iYgTgZObu1dk5g1d9iPNF+ZOGWaONDMzpwwzR5qZmVOOuaNSxmnAcyG9aehfAI/KzG0dt1S9iFgM
3JKZx3TdizQfmTvtMnOk3TNz2mXmSLtn5rTP3FFpY3ENnog4ADg2My8BLqV3mJuGlJkTwPqIOLTr
XqT5xtxpn5kj7ZqZ0z4zR9o1M6cMc0eljcWAB/g14FPN5+cDr+qwl3GzL3BLRHwpIi6evHXdlLoV
EadFxMqu++iYuVOGmaP7MXMAM6cUM0f3Y+YAZk5J5o7up63cGYtTtCJiHfCczPxBc/8m4HmZ+b1u
O6tfRPz3mZZn5mWj7kXzQ0QcBdwGvD4zz+66n66YO2WYOZrOzOkxc8owczSdmdNj5pRj7mi6NnOn
+gFPROwDvDgzP9y37FnAj71YldS+iHhX8+mzM/PxnTbTEXNHGh0zx8yRRsnMMXOkUWszd6o/RSsz
fwLcPG3ZPwN7dtPReIiIrzYfN0TEPX23DRFxT9f9qRvNheFeBPwJ8NOIOL7jljph7rTPzNFMzJwe
M6d9Zo5mYub0mDllmDuaSdu5U/2Ap/GBAZdpQJl5cvNxVWau7rutyszVXfenzjwX+FpmbqD3tpmv
7LifLpk7LTJztAtmzhQzp0VmjnbBzJli5rTM3NEutJo7S1ppqSMRcRLwJODAiHhz35dWA4u76Wr8
RMTJwMMz8/zmivqrMvPbXfelTrwSeF/z+UXAuyLizMzc0mFPI2XulGfmqI+ZY+YUZ+aoj5lj5oyE
uaM+reZO7UfwLANW0htUreq73QO8sMO+xkZEvAP4X8BZzaJlwCe660hdac7H3iczLwfIzE3Ap4Gn
d9rY6Jk7BZk5mmTm7GDmFGTmaJKZs4OZU5i5o0klcmccLrK8GLgwM1/QdS/jKCJuBE4Ars/ME5pl
azPzuG47k7pj7pRj5kj3Z+aUY+ZI92fmlGXuqKSqT9ECyMyJiHhI132MsS2ZmRGRABGxV9cNafQi
4sTdfT0zrx9VL/OBuVOUmSMzZxozpygzR2bONGZOceaOiuVO9QOexo0RcTHwd8DGyYWZ+dnuWhob
F0bEh4F9IuI3gFcAH+m4J43enzUfVwBrgJuAAI4DrgVO6qivLpk7ZZg5AjNnJmZOGWaOwMyZiZlT
jrkjKJQ71Z+iBRAR58+wODPzFSNvZgxFxLOAZ9N7wn2xeZtELUAR8VngHZm5rrn/aOAPMnPBnZNt
7pRj5miSmTPFzCnHzNEkM2eKmVOWuaNJbefOWAx4VF5ErKbviK/MvKvDdtSRiLglM499oGXSsMwc
gZmj0TFzBGaORsvcEbSfO2NxilZErKD39mLH0jvECQAnzMOLiFcD7wQ2AdvpTZkTOLKF2suARzR3
12fm1mFrqri1EXEuU1f6fymwtsN+OmPulGHmaBozp2HmlGHmaBozp2HmlFMqd8ycarWaO7W/Tfqk
vwYOBk4FLgMeCmwYpmBEPCgizouIS5r7j4qIVw7daX3OBB6dmYdn5pGZeURmtrHTcwrwb8BfAR8E
bo+Ipw5bV8W9HLgFeGNz+0azbCFqNXfMnB3MHPUzc6a4r1OGmaN+Zs4UM6ec1nPHzKlaq7kzFqdo
RcQNmXnC5NvLRcRS4IrMfOIQNS8Bzgd+LzOPj4glwA2Z+Zi2+q5BRHwBeH5m3tty3euA0zNzfXP/
EcCnMvNxba5HKqXt3DFzeswcaWbu65Rh5kgzM3PKKZE7Zo4mjcUpWsDk4Wc/aS5K9CPgoCFrHpCZ
F0bEWQCZuS0iJoasWaOzgKsi4hpg8+TCzHzDkHWXTgZQU+/25h+OoUTEYcDDM/PSiNgDWJKZQ/1v
g6ZExJOBPwAOY+dzhof+384KtZ07Zk5PVZkD5k5JZs5O3Ncpw8zRDmbOTsycckrkjplTqbZzZ1wG
POdExL7A24CLgZXA7w9Zc2NE7E/vfEgi4onAT4esWaMPA/8CrKN3jmhbrp3hXMNrhykYvbcZPAPY
DziK3qGkZwPPGKaudnIe8CbgOmAh/oPcr+3cMXN6qskcMHdGwMyZ4r5OGWaO+pk5U8ycckrkjplT
r1ZzZ1xO0ToiM7/9QMtmWfNE4APAo4GbgQOBF2XmTUM1W5nJwzML1F0O/BZwcrPoCuCDmbl519/1
gDVvBB4PXDPZc0SsW2iHfZYUEddk5hO67mM+aDt3zJyemjKnqWvuFGTmTHFfpwwzR/3MnClmTjkl
csfMqVfbuTMuA57rM/PEacuuG+acw+ZFMgEcTe/K5uuBRcO+SGoTEX8MfAf4HDsfQjjnt/GLiMXA
xzPzpUM3uHPdazLzCX3nDC8Brs/M49pcz0IWEe8BFgOfZefnw/WdNdWRtnPHzOmpKXOa2uZOQWbO
FPd1yjBz1M/MmWLmlNN27pg5dWs7d6o+RSsijqH31n17R8Tz+760mr6385ujq5tQu6VvfdcDJ+76
W8bSS5qPZ/UtG+pt/DJzIiIOi4hlmbllqO52dllE/C6wR0Q8C3gtveBUeyany2v6liXw9A566UTB
3DFzemrKHDB3SjNz3NcpzcxRPzPHzBmFVnPHzKleq7lT9YCH3vT3ecA+wC/2Ld8A/MZcCkbEwcAh
9J7EJ9CbLkMv1Pace6t1yswjCpX+FnBlRFwMbOxb3/uGqPlW4JX0zmd9NfCPwLnDNKmdZebTuu5h
Hmg1d8ycnVWWOWDuFGXmAO7rFGXmqJ+ZA5g5xRXKHTOnUm3nzriconVSZl7dUq1fB15Gb4L2r0wF
0Abggsz8bBvrme8i4umZ+S/TJvc7DLsdIuIdu6j7zmHqqqyIeBDwx8BDMvPnI+JRwEmZeV7HrY1c
W7lj5vSYOZqJmTPFfZ12mTmaiZkzxcxpX8ncMXPq1XbujMuA50+BdwH3AV8AjgPelJmf2O037r7m
CzLzMy21WJ2IeGdmviMizp/hy5mZrxiy/oltn88cEd+muSp/vxzyrS2bbTBT3aG2QY0i4hLgfOD3
MvP45jzcGxbihdbazh0zp77MaeqaOwWZOVPc12mXmXO/umYOZk4/M6d9JXPHzKlX27lT+ylak56d
mW+JiNPoXbDq+cDlTL1N3Fw8NCJW05ssf4TeuaFvzcx/GrbZGjThswi4JDMvLLCKP2sO1/w08LeZ
eXMLNfvPW1wBvIjeW/oN6x+m1T0N+OGwRSPiKcBVmTnRt6xIOLfogMy8MCLOAsjMbRGxUN9GtO3c
MXPqyxwwd0ozc6a4r9MiM+d+zJweM2eKmdOywrlj5lBl5kDbuZOZ1d+AW5qP5wLPaT6/aciaNzUf
TwUuonexseu7/lk72LbXFqx9MPAG4Ep653W+rcA6ritQcxG94Bi2zr3AZcBBfcvm9XMM+Aqw/2Sf
wBOBy7ruq6Nt0WrumDk7tkPVmdOsx9xp7+c2c6a2hfs6ZbarmTNzTTMnzZzmo5nT/rYtkjtmTn2Z
0/TXau6MyxE8n4uI2+gdQviaiDgQ2DRkzclzQ59L723nbomI2N03jKlLI+JM4G/Z+YJdc3770L4a
PwLeHxFfBt4CvJ3eoaBzEhH9V+BfRG/iXOI5/nDgoBbqrAfeS+/q9K/MzKuYet7NV28GLgaOiogr
gQOBF3bbUmfazh0zp6eazAFzZwTMnCnu65Rh5szMzDFzzJxyiuSOmQPUlznQcu6MxTV4ACJiP+Cn
2XubuL2AVc2TfK71zqd3tfcjgOPpvTf9VzLzca00XInmvMvpMoc/7/KRwIuBFwB30gu4z2Tmfw5R
88t9d7fRO5z0/2Tm+iFaJSI20DtHNJqPPwLOyiHPIY6I6zPzxIh4OL2f/6PAK7L39pHzVnNe6NH0
tsf6zNzacUudaTN3zJyemjKnqWvuFGbmTHFfp31mzo66Zk7DzJli5pRRInfMnB11q8scaDd3qh/w
RMSewMMz86a+ZYcCE5n5gyHqLgIeC3wrM38SEfsDh2Tm2qGbFhFxNb0X3YWZOfT5ljWKiBsy84Tm
85X0Auj5mTkvj6wr9VqrUYltYeaUZeb01JQ7Zs4U93XqY+b0mDl1MnPqY+b01JQ5UOhvijEY8CwF
bgOOy8yNzbJ/An43M68dom4ALwWOzMw/bDb0wZn59SH7LVK3qX088JTm7hX9T5Qhaq4AXgucTG+6
egVwdmYOe4hm6yLizbv7ema+b451J39nR2TmH7X5O5thXYdm5r+3XbcNpV5rNSqxLcycHTWryRww
d0oyc6a4r7Ojrplj5hRj5kwxc3aqvaBzx8wpq8RrbVGL/XWiOXzpIuB/wI6J14EtBPEHgZOAlzT3
NwB/NZdCEXFyRCxuu+60dbwR+CS9cxcPAj4REa8fti7wcXoXQPsA8JfN5389RJ8XNh/XRcTavtu6
iBh2er8GeA29Qz8PAX6T3tX5VzW3uZr8nZ3e3B/qdxYRb2k+fiAi3t9/A84cos+iCr7WqlNoW5g5
PTVlDpg7xZg5U9zXMXP6mDmFmDlTzJwd65j3uWPm9NSYOVDotZbz4MrRw96AY4DLm8/fBryhhZqT
V7G+oW/ZnK4cDzwJOKftutPWsRbYq+/+XsDaFup+Y5Bls6j34ObjYTPdhuz1cnrnBk/eXzX5vJgv
z4Xme+9sPv428OvTb8P2W/JW4rVW663tbWHm7KhTTeY0dc2dgjczp+y2qCl3zJwd9c2cgjczp+y2
qClzmhrzPnfMnB3fW2XmND23+lqbl+eizVZm3hY9jwB+lanD6IaxtZkKJ0D0rhy/fY79XRUR97Zd
d5oAJvruTzTLhnV9RDwxM78GEBFPAOY8UczMO5qP322ht+keBGzpu7+lWTastn9n/xERDwFeDpxC
O7+nkSj0WqtSgW1h5vTUlDlg7hRl5kxxX8fMaZg5BZk5U8wcoILcMXN2qDJzoP3X2lgMeBrnAecC
6zLz7hbqvZ/e4VIHRcS76b1V2dvmWiwzbyxRt8/5wDURcVFz/1fobZNhPQ64KiImz1s8FFgfEevo
Xe39uNkUi6krpt/vS0291UP0+nHg69O2wQVD1JvU9u/sQ8CXgCOB6/qWT15Ffqh37tiViDg4h3jn
gz5tv9Zq1ua2MHN6asocMHd2ycwpYiHv65g5PWbObrSUO2bOlIWcOVBB7pg5O9ScOdDia636iyxP
it4VqO8AXpCZl7ZU8xjgGfSeGF/KzFvned0T6V2sC3oXAbuhhZqH7e7rBafFc9Jsg8mp5+VtbIOm
buu/s4j4UGa+ZujmBl/f5zPzF1qo0/prrVZtbwszp77MAXNnN+syc1q20Pd1zJweM2e36xs6d8yc
KQs9c5q6Cz53zJzdrm/e7euMzYBHkiRJkiRpoar+XbQkSZIkSZIWurEb8ETEGda1bqma1i1ft0Y+
f61bY92aei1Zt0a1bWOfv9atsa6ZM6Wm35t1y9W0bh11x27AA5QKY+vWV7emXq1bN5+/1q2xbk29
lqxbo9q2sc9f69ZY18yZUtPvzbrlalq3grrjOOCRJEmSJElaUKq4yPKyWJ4rYq+BHrs1N7M0lg/0
2AOP3TxwD/fctY3V+w32rvL/dfNg6wfYymaWMvjjS9SNxYsHrrslN7EsVsy1rd3UvY9lsccDPi4n
JgauOR+2rXV7NnD3jzPzwNabKGhZLM8VDJg7s9gWP/eYnw30uLvv2s6++w0+g//eupUDPW4+PB8A
YunSgR63Zft9LFv0wNkAkFu3Drz++bIduqxbU6+zrbuJjWzJzdF6EwWVypyHPGbjwD385M7t7LP/
YLlzx+37Dlx3y8R9LFs84Ot485aBHjcfnmfWHe+64585K2bx99Umlg66/7/n4H8nbN26kaVLB+zh
4MH/Zt12z70sWb3nQI9d+q1NA9edzd+ZsWg2f18N9ncQs/i7fTZ/s/3/7d17sF1neR7w55V0ZNmW
Lzg2AYJB2IANuGCwIFwCMZcODqUtTtyBuJMWWtAQQ7h0YErSFpg0BhpmYAK0psYF00JpBiiBQAkN
TAhgKGDAYIMxEBsSLiUE8A0jWTrn6x86KrKw5HNZ39pa5/x+Mx7vvc4+z/q2zj6P1n619t5tYWHJ
ucv6Xaul/0os5zFWm5bxZ7us48g9S849HLrsYM+vljaxmLEtdXQevumJg+c+6z1fGzwzSS4+7dQu
uak+J1xtPO7Y4UPb+meaZQAAGJdJREFU0otiOeavv6FLLn19uL3rsPq4x6XYkqPzy/X4wXNf8/5P
DZ6ZJP/qXo/skrucg4nl2PSLdxs8c893vjt4JtP06faRWS9h2Xp1zkvf9/nBM5Pkwiee1yV3/hvX
dcnt1WWQTLRz6ug8fO6cwXPbA08fPDNJvv3ipf8j73Kc/NRruuRu2Lq0wdVyLGcAsRwLt9zSJbc2
b+6Su/GEpf8Dw3Ls+f4PuuT2el784YV33u7zKy/RAgAAAJg4Ax4AAACAiTPgAQAAAJg4Ax4AAACA
iTPgAQAAAJg4Ax4AAACAiVvRgKeqjq+qC/a7fnZVvf8gt72kqu6/0gUC6BxgTDoHGJveAYaw0jN4
jk9ywR3eKklr7Zmtta+scD8Aic4BxqVzgLHpHWDVVjrgeVWSU6vqiqp69eK2rVX1rqr6alW9vaoq
Sarqo1W1vao2VtWlVXVVVV1ZVS8c5B4A64HOAcakc4Cx6R1g1Tat8PtekuSM1tqZyd5TCJM8OMkD
knw3yWVJHpXkE/t9z5lJfqm1dsbi9xx/qB1U1Y4kO5JkS45a4TKBNaJ75yzeRu8Aic4Bxuf5FbBq
Q77J8mdaa99urS0kuSLJtgO+fm2SU6rq9VV1TpIbDxXWWru4tba9tbZ9ro4YcJnAGjFo5yQH9E70
DnAbOgcYW8fnV1v6rBiYqSEHPLv2uzyfA84Oaq39OMmDknw0ybOTXDLgvoH1R+cAY9I5wNj0DrAs
K32J1k1JjlnON1TViUluba29u6quSfK2Fe4bWH90DjAmnQOMTe8Aq7aiAU9r7YdVdVlVXZXkg0k+
sIRv+6Ukb6mqfWcN/e5K9g2sPzoHGJPOAcamd4AhrPQMnrTWzj9g00f3+9pz97t89n63echK9wes
bzoHGJPOAcamd4DVGvI9eAAAAACYAQMeAAAAgIkz4AEAAACYOAMeAAAAgIkz4AEAAACYuBV/itZa
cMl5T+qUfE2X1Jrr8+M69v01eOZNT906eGaS5Pob+uTCAWrTxmy80y8Mnvuip/zLwTOT5PvPPb5L
7t3e0afPfvjYewye+Quf6NORe775111yU8N3b5KktT65U1vvxNSRW7LhvqcPnnvhI+48eGaS1Kad
XXI33vmkLrnfO+/eg2fO3dznsXunt36qS24vNbe5S27bs7tLbhcTrLHauDEbjj9u8Nz25esGz0yS
W244rUtube7z+N25ffjOOfLL3xk8M0kWbrmlS2679dYuuQs/vr5Lbm3oc5zT9oxbEM7gAQAAAJg4
Ax4AAACAiTPgAQAAAJg4Ax4AAACAiTPgAQAAAJg4Ax4AAACAiTPgAQAAAJi47gOeqvpk730A7KNz
gLHpHWBMOgc4mO4DntbaI3vvA2AfnQOMTe8AY9I5wMGMcQbPzYv/v2tVfayqrqiqq6rq0b33Daw/
OgcYm94BxqRzgIPZNOK+zk/yodbahVW1MclRh7pxVe1IsiNJthz6pgC3Z1mdkxzQOxu2dl4esAat
/Fhn7tgRlgesMSvvHMc5sCaNOeD5bJI3V9Vckj9prV1xqBu31i5OcnGSHLvhhDbC+oC1ZVmdk9y2
d46bO0nvAMu14mOd4466m84BlmvlnTN3Z50Da9Bon6LVWvtYksck+U6SS6vqn421b2D90TnA2PQO
MCadAxxotAFPVd0zyfdba29KckmSh4y1b2D90TnA2PQOMCadAxxozJdonZ3kxVW1O8nNSUyYgZ7O
js4BxnV29A4wnrOjc4D9dB/wtNa2Lv7/rUne2nt/wPqmc4Cx6R1gTDoHOJjRXqIFAAAAQB8GPAAA
AAATZ8ADAAAAMHEGPAAAAAATZ8ADAAAAMHFjfkz6yrWkzc8Pn3v1tcNnJklrXWI33u0uXXI/c8Xw
ucc9ZePgmUly5zd8p0suHKjtmc/8D380eO7G6jNXv8tFX+uSu/vhZ3TJPfKff2/wzPbnOwfP7KrT
3xXdTG29E9N+ujMLV3191suYuff89ae65J5794cNnllzmwfPTJKp/aa13bfOegmsSEsWhn9+VZvn
Bs9MktOf+5UuuRtOuFOX3BtOHb4fbjr5XoNnJskJl36/S+7GY47pkptOj7GaX+iSO3/99V1yD/aX
hTN4AAAAACbOgAcAAABg4gx4AAAAACbOgAcAAABg4gx4AAAAACbOgAcAAABg4lY14Kmq46vqgv2u
n11V71/9sgB+ns4BxqRzgLHpHWA1VnsGz/FJLrjDWwEMQ+cAY9I5wNj0DrBiqx3wvCrJqVV1RVW9
enHb1qp6V1V9tareXlWVJFV1VlX9ZVV9rqo+VFV3XeW+gfVH5wBj0jnA2PQOsGKrHfC8JMlftdbO
bK29eHHbg5O8IMn9k5yS5FFVNZfk9UnOa62dleTNSS5c5b6B9UfnAGPSOcDY9A6wYps6ZH6mtfbt
JKmqK5JsS3J9kjOS/PniwHljku8dKqSqdiTZkSRbclSHZQJrxCCds/j9ege4IzoHGNvwz682bO24
XGBWegx4du13eX5xH5Xky621Ryw1pLV2cZKLk+TYOqENukJgLRmkcxK9AyyJzgHGNvjzq+PmTtI5
sAat9iVaNyU5Zgm3uybJSVX1iCSpqrmqesAq9w2sPzoHGJPOAcamd4AVW9WAp7X2wySXVdVV+70J
2O3d7tYk5yX5D1X1xSRXJHnkavYNrD86BxiTzgHGpneA1Vj1S7Raa+cfsOmj+33tuftdviLJY1a7
P2B90znAmHQOMDa9A6zUal+iBQAAAMCMGfAAAAAATJwBDwAAAMDEGfAAAAAATJwBDwAAAMDEGfAA
AAAATNyqPyZ9NK0NH7n71sEze9pz3be65F577nsHz3zic84cPBNGVUlt3Dh47ML1Nwye2dPG/3NV
l9zX3edjg2e+8EdnD54Jo1qYn/UKlq6qS+y5J/9yl9wPffcLg2eec8+HDZ4JY2l75jP/wx/Nehkz
t3DLLV1yP/eyDwye+cS7nzV4ZpIuz7OTZP7GG7vkcmjO4AEAAACYOAMeAAAAgIkz4AEAAACYOAMe
AAAAgIkz4AEAAACYOAMeAAAAgIlb9oCnqp5XVVdX1dur6h9V1UuW8b3bqur85e4TWN/0DjAmnQOM
SecAQ9m0gu+5IMkTWmvfXrz+vgNvUFWbWmt7bud7tyU5P8l/X8F+gfVL7wBj0jnAmHQOMIhlDXiq
6o1JTknywap6c5IfJ9neWntuVV2aZGeSBye5rKrem+SPFr+1JXlMklcluV9VXZHkra211w5zN4C1
Su8AY9I5wJh0DjCkZQ14WmvPrqpzkjy2tfZ3VfX0A25y9ySPbK3NV9WfJnlOa+2yqtqaveX0kiQv
aq09+Y72VVU7kuxIki05ajnLBNYQvQOMSecAY9I5wJCGfpPld7bW5hcvX5bkNVX1vCTHH+SUwoNq
rV3cWtveWts+lyMGXiawhvTpndI7wO1yrAOMSecASzb0gOcn+y601l6V5JlJjszeUwpPH3hfAIne
Acalc4Ax6RxgyVbyJstLUlWnttauTHJlVT00yelJ/ibJMb32CaxvegcYk84BxqRzgDsy9Bk8+3tB
VV1VVV9KsjvJB5N8Kcl8VX2xql7Ycd/A+qR3gDHpHGBMOgc4pGWfwdNa27bf5UuTXLp4+ekH3O53
DhLxuOXuE1jf9A4wJp0DjEnnAEPpeQYPAAAAACMw4AEAAACYOAMeAAAAgIkz4AEAAACYOAMeAAAA
gIlb9qdoMUNVXWLPucf2wTPf8+1PDp6ZJOfe/WFdcuHntKTt2TPrVcxep955wbZHDp75oe9+ZvDM
JHni3c7skgu3UZWa2zx4bNt96+CZPW3YurVL7q/de/jOefk1Hx88M0ledspZXXKzYWOf3LbQJ7c6
/Tv0wnyf3Cnq9Hd8F611ia1NfZ4O/9p9HjV45iv+6i8Gz0yS37tXp+dXU3p8JanNw/8dnCRt164u
uQfjDB4AAACAiTPgAQAAAJg4Ax4AAACAiTPgAQAAAJg4Ax4AAACAiTPgAQAAAJg4Ax4AAACAiTvk
gKeqtlXVVUPsqKq+WVUnDpEFrE06Bxib3gHGpHOAnpzBAwAAADBxSxnwbKqqt1fV1VX1rqo6Kkmq
6vFV9YWqurKq3lxVRxxq+z5VdWRVfbCqntXh/gDTp3OAsekdYEw6B+hiKQOe05L8p9ba/ZLcmOSC
qtqS5NIkT22t/b0km5L89sG275e1NcmfJnlHa+1Nh9ppVe2oqsur6vLd2bXMuwVM2Ew6J9E7sI7N
/lin7Rz6PgGHr9l3juMcWJOWMuD5m9baZYuX35bkV7K3lK5rrX1tcftbkzzmENv3eW+St7TW/usd
7bS1dnFrbXtrbftcjrijmwNrx0w6J9E7sI7N/lintgxxP4BpmH3nOM6BNWkpA552B9eX47Ik51RV
rSIDWNt0DjA2vQOMSecAXSxlwHOPqnrE4uXzk3wiyTVJtlXVvRe3/1aSvzzE9n1emuTHSf7jahcO
rFk6Bxib3gHGpHOALpYy4LkmyXOq6uokd0pyUWttZ5JnJHlnVV2ZZCHJGw+2/YC85yc5sqr+cKg7
AawpOgcYm94BxqRzgC42HeqLrbVvJjn9IF/7SJIHL2P7tv2uPmM5iwTWB50DjE3vAGPSOUBPSzmD
BwAAAIDDmAEPAAAAwMQZ8AAAAABMnAEPAAAAwMQZ8AAAAABM3CE/ReuwUjXrFSxda7NewbK0PXsG
z3zMy58/eGaSnHT0l7rkfu2N9+2Se59/8eUuuWkLfWI7PBYmrUfvTKwfeq1355MfNnjmP/iVbYNn
JsmGM7Z0yb32aXfqknvKK77YJbc2z3XJnb/hxuFDJ/ZrliRpLW33rbNexdJ16oaFm27qktvDy045
a9ZLWJbr3n5Gl9xNVx3dJffkCz/ZJZf9TO2YpINex749cn/v1EcMnpkkqT7PK6575cO75C7M9Xnc
3uff9XmeOfZvmTN4AAAAACbOgAcAAABg4gx4AAAAACbOgAcAAABg4gx4AAAAACbOgAcAAABg4gx4
AAAAACZuJgOeqvrkLPYLrF96BxiTzgHGpHOAZEYDntbaI2exX2D90jvAmHQOMCadAySzO4Pn5lns
F1i/9A4wJp0DjEnnAEmyadYLOJiq2pFkR5JsyVEzXg2wHugdYEw6BxiTzoG177B9k+XW2sWtte2t
te1zOWLWywHWAb0DjEnnAGPSObD2HbYDHgAAAACWxoAHAAAAYOIMeAAAAAAmblYfk751FvsF1i+9
A4xJ5wBj0jlA4gweAAAAgMkz4AEAAACYOAMeAAAAgIkz4AEAAACYOAMeAAAAgInbNOsFLEVt3JCN
W48ZPHf+xhsHz5yiDVu2DJ554n/7/OCZSbLhpBO75J72ipu75H71NQ/uknu/136/S25a65N7bZ/Y
7nr9eZAtH/js4Jl7ev28qrrEnvIHR3TJfcM1H+6S+5z7Pr5Lbsq/NSVJqlJHDP+YaLt2DZ7Jog0b
++S2hS6xR33q6C65N5yxu0vuhqP7rLc2bx4+84ZOj4WedM70dOqGTfc8uUvufV7/rS651/+X4Z+7
JsmGXzypS25d32nm8KPb3+yoCgAAAGDiDHgAAAAAJs6ABwAAAGDiDHgAAAAAJs6ABwAAAGDiDHgA
AAAAJs6ABwAAAGDiNvUMr6qXJ7k5ybFJPtZa+/ABXz87yYtaa0/uuQ5gfdA5wNj0DjAmnQMcStcB
zz6ttZeOsR+AROcA49M7wJh0DnB7Bn+JVlX9m6r6WlV9Islpi9surarzFi+fU1VfrarPJ/n1ofcP
rC86Bxib3gHGpHOApRp0wFNVZyV5WpIzkzwpyUMP+PqWJG9K8g+TnJXkLofI2lFVl1fV5bcu7Bxy
mcAaMWTnLN7+//fO7uzqs2hg0nod6+xujnWAn6dzgOUY+gyeRyd5T2vtltbajUned8DXT09yXWvt
6621luRtBwtqrV3cWtveWtu+ecOWgZcJrBGDdU5y296ZyxGdlgxMXJdjnblyrAPcLp0DLJlP0QIA
AACYuKEHPB9L8pSqOrKqjsneUwX399Uk26rq1MXrvznw/oH1RecAY9M7wJh0DrBkg36KVmvt81X1
x0m+mORvk3z2gK/vrKodST5QVbck+XiSY4ZcA7B+6BxgbHoHGJPOAZZj8I9Jb61dmOTCQ3z9z7L3
taIAq6ZzgLHpHWBMOgdYKu/BAwAAADBxBjwAAAAAE2fAAwAAADBxBjwAAAAAE2fAAwAAADBxg3+K
Vhdzc8nd7zJ87tU3DZ+ZJK1NKndh167hQzuttR19ZJfc+Wuu7ZJ72n/e2CX3G8+4a5fce7230+9E
nz/evo7aknrAGYPHtsuvGjyTRVVdYjced2yX3IWf7uyS+5x7P65L7rX//iFdck+4avi/L+Y/cNng
md0dtSULDxr+Q3DqU18cPJNFC/NdYmtuc5fcu1x0eZfcu3U6NvveMx7UJXfX8R0yL/ng8KGd1RGb
s2HbyYPnzl/zjcEzu+p07NBFr+dX19/QJXfhJz/tknvMkxe65P7gnz60S+4PH9zneVuef/ubncED
AAAAMHEGPAAAAAATZ8ADAAAAMHEGPAAAAAATZ8ADAAAAMHEGPAAAAAATZ8ADAAAAMHEGPAAAAAAT
Z8ADAAAAMHEGPAAAAAATd9gOeKpqR1VdXlWX3zp/y6yXA6wD+/fO7j16B+jrNp2z+yezXg6wxt3m
+ZXjHFiTDtsBT2vt4tba9tba9s0bj5r1coB1YP/emdukd4C+btM5c0fPejnAGneb51eOc2BNOmwH
PAAAAAAsjQEPAAAAwMTNfMBTVZdU1fZZrwNYH3QOMDa9A4xJ58D6tWnWC2itPXPWawDWD50DjE3v
AGPSObB+zfwMHgAAAABWx4AHAAAAYOIMeAAAAAAmzoAHAAAAYOIMeAAAAAAmrlprs17DHaqqHyT5
1hJvfmKSv+uwDLnTy53SWtd67j1bayd1WEM3h0HvHA4/N7lrO3dKa11urs5ZmSnlTmmtcqeZq3N+
Zko/N7n9MuUeXrm32zuTGPAsR1Vd3lrbLlfulNYqd9o8fuVOMXdKa+2ZO0VT+zP2+JU7xVyd8zNT
+rnJ7Zcpdxq5XqIFAAAAMHEGPAAAAAATtxYHPBfLldsxU27/3Cny+JU7xdwprbVn7hRN7c/Y41fu
FHN1zs9M6ecmt1+m3Ankrrn34GF8VXVza23rftefnmR7a+25A2R/NMmLWmuXH7D90iS/muSGxU1P
b61dsdr9AYe/GXVOJfmDJP8kyXySi1prr1vt/oDD34w65+NJjlm8euckn2mtPWW1+wOmYUa98/gk
r87ek0Buzt7nV99Y7f4Y16ZZLwBW4cWttXfNehHAuvD0JCcnOb21tlBVd57xeoA1rLX26H2Xq+rd
Sd47w+UA68NFSf5xa+3qqrogyb/N3uMfJmQtvkSLw0hVnVRV766qzy7+96jF7Q+rqk9V1Req6pNV
ddri9iOr6n9U1dVV9Z4kR870DgCT0rFzfjvJ77fWFpKktfa3o9wh4LDW+zinqo5N8rgkf9L9zgCT
0LF3WpJjFy8fl+S73e8Mg3MGD0M4sqr2f3nUCUnet3j5j5K8trX2iaq6R5IPJblfkq8meXRrbU9V
PSHJK5L8RvY+ibqltXa/qnpgks8fYr8XVtVLk3wkyUtaa7uGvVvAYWoWnXNqkqdW1blJfpDkea21
rw9+z4DD0ayOc5LkKUk+0lq7ccD7Axz+ZtE7z0zyv6rqp0luTPLwwe8V3RnwMISfttbO3Hdl32tE
F68+Icn99759RZLk2Kramr1T4bdW1X2yd1o8t/j1xyR5XZK01r5UVV86yD5/N8n/TbI5e9+M6l8n
+f2h7hBwWJtF5xyRZGdrbXtV/XqSNyd59EFuC6wts+icfX4zySVD3AlgUmbROy9M8qTW2qer6sVJ
XpO9Qx8mxICH3jYkeXhrbef+G6vqDUn+orV2blVtS/LR5YS21r63eHFXVb0lyYtWv1RgDejSOUm+
neR/Ll5+T5K3rG6ZwBrRq3NSVScmeViSc1e/TGANGbx3quqkJA9qrX16cdMfJ/mzQVbLqLwHD739
7yS/s+9KVe2bRB+X5DuLl5++3+0/luT8xduekeSBtxdaVXdd/H9l7+nLVw25aGCyunRO9r7/xWMX
L/9qkq8Ns1xg4np1TpKcl+T9Bz6JA9a9Hr3z4yTHVdV9F6///SRXD7dkxmLAQ2/PS7K9qr5UVV9J
8uzF7X+Y5JVV9YXc9kyyi5Jsraqrs/clV587SO7bq+rKJFcmOTF7P74YoFfnvCrJbyz2zivjlGVg
r16dkyRPS/KODmsGpm3w3mmt7UnyrCTvrqovJvmtJC/ueB/opFprs14DAAAAAKvgDB4AAACAiTPg
AQAAAJg4Ax4AAACAiTPgAQAAAJg4Ax4AAACAiTPgAQAAAJg4Ax4AAACAift/CrZMYNGtzhcAAAAA
SUVORK5CYII=
"
>
</div>

</div>

</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="prompt input_prompt">In&nbsp;[&nbsp;]:</div>
<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span> 
</pre></div>

    </div>
</div>
</div>

</div>
    </div>
  </div>
</body>

 

<script type="application/vnd.jupyter.widget-state+json">
{"00d1991457d541d48c1ca5aad7fe3e8f": {"model_module": "@jupyter-widgets/controls", "model_name": "IntProgressModel", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "IntProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "info", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_365509bb19e3492ca4cf0467a65677cf", "max": 1, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_9ebf269ea639433484a26cffb896048f", "value": 1}}, "02fa22f70da849ceb5cef6c6a9a6c42f": {"model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "056d566b9dfc4c12b13b0fcb6075a270": {"model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "06d9c755ed0e4758a602472ba8a7c886": {"model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "0e04ea39154b40819a6ae5b5914847a2": {"model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_e65cf9ab4221419e97f74426d0ee292f", "placeholder": "\u200b", "style": "IPY_MODEL_21fec75ed3724763aa3cd93b6e2c7ab9", "value": " 68% 35326/51785 [00:00&lt;00:00, 87505.93 examples/s]"}}, "1a1013a121c6452a9dcf5be7d91b6bb6": {"model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": ""}}, "1a6d255cf1744383b9d216d4a04f0acf": {"model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": ""}}, "1f0d1a90ce324a4c946574e4dc1affba": {"model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "initial"}}, "2121f86cde9c4eefb6b465d07b9e01df": {"model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "21fec75ed3724763aa3cd93b6e2c7ab9": {"model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": ""}}, "24187cb2ff0e45f9833a8c48c96efcc7": {"model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "245051a8a9e746568bc24399fded1713": {"model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "2966a7701dde4f46885083de9c6b1eb0": {"model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_84890da093d9455b8bfc29a6f432bc8f", "placeholder": "\u200b", "style": "IPY_MODEL_1a6d255cf1744383b9d216d4a04f0acf", "value": "1066 examples [00:00, 5043.56 examples/s]"}}, "2b125be6a55f43798366013f4d4ee840": {"model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "34171b17684e49708e0633eed1c8216c": {"model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_ed874e7a5d294e28940f3ac0dcc99a96", "IPY_MODEL_9bf6fd00b1b64af2ba4b1574f0e66bc0"], "layout": "IPY_MODEL_d8b58951df5c4dfeb6a1d24669b522ff"}}, "365509bb19e3492ca4cf0467a65677cf": {"model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "39e1c098df36444c932002921ed9bd51": {"model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "3dbe4b29ee9d467cbeef39946925db10": {"model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "4469976e9ed0411eb472ce953b2b942f": {"model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": ""}}, "46ec15c44cf0445eb7097b9ffcd48b72": {"model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "5599b46c0a374da4a1bc3566747f395c": {"model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "5d733f7bd7f245ce8e92cd7a74c9a967": {"model_module": "@jupyter-widgets/controls", "model_name": "IntProgressModel", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "IntProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "danger", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_b1f9c517f8404821aa7e374d674982e8", "max": 1803, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_7dc0f107d2be48de8be39dfbaff50c64", "value": 0}}, "5ffbbf72f5dd402ca28cd2d35d79295b": {"model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": ""}}, "6276134a3fa7469faa2cdb9b9841a236": {"model_module": "@jupyter-widgets/controls", "model_name": "IntProgressModel", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "IntProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "Dl Size...", "description_tooltip": null, "layout": "IPY_MODEL_b234ad53c4f048b3b336d9a3e5441189", "max": 1, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_1f0d1a90ce324a4c946574e4dc1affba", "value": 1}}, "66234e0238854b9cb36cb57a1f2aa44f": {"model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_bb6f1ba6e0cf46889984cfeb9bd16097", "placeholder": "\u200b", "style": "IPY_MODEL_5ffbbf72f5dd402ca28cd2d35d79295b", "value": "  0% 0/1803 [00:00&lt;?, ? examples/s]"}}, "66669c75d14146139cba84f57a4c12ac": {"model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "68c2817c35724d908b2dc27ddbc8d65a": {"model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "76125a19d0e24b21a3e0cae93fe34b60": {"model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_2121f86cde9c4eefb6b465d07b9e01df", "placeholder": "\u200b", "style": "IPY_MODEL_80bca50374f8438bacd78207ad6c8e39", "value": "1159 examples [00:00, 5181.69 examples/s]"}}, "792f241ea8394e9ca81eeac6909481f8": {"model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "7dc0f107d2be48de8be39dfbaff50c64": {"model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "80bca50374f8438bacd78207ad6c8e39": {"model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": ""}}, "84890da093d9455b8bfc29a6f432bc8f": {"model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "84e50d0a4e824d409249d1c05f13a8fc": {"model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "initial"}}, "8543fc0a11d841bb9593d942eaf05601": {"model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_eaa4b7905b524f5ea5e2d5c73437e0d5", "IPY_MODEL_76125a19d0e24b21a3e0cae93fe34b60"], "layout": "IPY_MODEL_c003c5f9ade94009a41a9d225bc12502"}}, "8636cf713e6548aca61d10c821199873": {"model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_39e1c098df36444c932002921ed9bd51", "placeholder": "\u200b", "style": "IPY_MODEL_89a784492dfb49a1a83aa693ef82a7b1", "value": "51742 examples [00:08, 7022.27 examples/s]"}}, "89a784492dfb49a1a83aa693ef82a7b1": {"model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": ""}}, "8a66272607e04f92b935580029abe3a1": {"model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_5d733f7bd7f245ce8e92cd7a74c9a967", "IPY_MODEL_66234e0238854b9cb36cb57a1f2aa44f"], "layout": "IPY_MODEL_d2077508138b42e688b069d59ba044a0"}}, "8e4d74fcb67d49378495b45510ff668c": {"model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_00d1991457d541d48c1ca5aad7fe3e8f", "IPY_MODEL_8636cf713e6548aca61d10c821199873"], "layout": "IPY_MODEL_985545ca832b43fbb75382850f1c39f1"}}, "90a547d5b93746ceae81fd001be9004f": {"model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": ""}}, "918b595ca82f4e889e8851e7a3301542": {"model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_db2ba1f0367d4dc4abcea8e19736a80f", "IPY_MODEL_e1050782a98a48c7ba72adacd240b84a"], "layout": "IPY_MODEL_06d9c755ed0e4758a602472ba8a7c886"}}, "985545ca832b43fbb75382850f1c39f1": {"model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "9bf6fd00b1b64af2ba4b1574f0e66bc0": {"model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_24187cb2ff0e45f9833a8c48c96efcc7", "placeholder": "\u200b", "style": "IPY_MODEL_90a547d5b93746ceae81fd001be9004f", "value": "  0% 0/1193 [00:00&lt;?, ? examples/s]"}}, "9ebf269ea639433484a26cffb896048f": {"model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "a74e3d4851ad40a9802864c7e9fa3313": {"model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_e44536ead67141a58ce0aaad0ede93d2", "placeholder": "\u200b", "style": "IPY_MODEL_1a1013a121c6452a9dcf5be7d91b6bb6", "value": "124/|/100% 124/124 [00:05&lt;00:00, 23.27 MiB/s]"}}, "a845cf55d3cc451cbf6023f53451b987": {"model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "ac26758ef636439ea359ddf0d1729743": {"model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "b1f9c517f8404821aa7e374d674982e8": {"model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "b20f695e03d748bdb2016a09c6712e6a": {"model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_edf5019ce88945bd955f66e539a68c55", "IPY_MODEL_e492e498b2da4c3c8be865af31caeac8"], "layout": "IPY_MODEL_68c2817c35724d908b2dc27ddbc8d65a"}}, "b234ad53c4f048b3b336d9a3e5441189": {"model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "bb6f1ba6e0cf46889984cfeb9bd16097": {"model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "bd7d5597bcc24a5ba6ed233ff6869af3": {"model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "c003c5f9ade94009a41a9d225bc12502": {"model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "c6c4fa9e09454d6f81d9e7cabc1ad9d8": {"model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_e522f0058696468a979a4232787e8057", "IPY_MODEL_0e04ea39154b40819a6ae5b5914847a2"], "layout": "IPY_MODEL_a845cf55d3cc451cbf6023f53451b987"}}, "c76e58b8968747bc915fc37594da12fb": {"model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "d0979a4394f84ddb89c4307a18df05ee": {"model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "d2077508138b42e688b069d59ba044a0": {"model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "d769fc5c617e4a9dbd5ae1b132b9981b": {"model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "initial"}}, "d8b58951df5c4dfeb6a1d24669b522ff": {"model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "db2ba1f0367d4dc4abcea8e19736a80f": {"model_module": "@jupyter-widgets/controls", "model_name": "IntProgressModel", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "IntProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "Extraction completed...", "description_tooltip": null, "layout": "IPY_MODEL_46ec15c44cf0445eb7097b9ffcd48b72", "max": 1, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_d769fc5c617e4a9dbd5ae1b132b9981b", "value": 1}}, "de36871931e84fef867700db40d0126c": {"model_module": "@jupyter-widgets/controls", "model_name": "IntProgressModel", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "IntProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "info", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_d0979a4394f84ddb89c4307a18df05ee", "max": 1, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_02fa22f70da849ceb5cef6c6a9a6c42f", "value": 1}}, "e1050782a98a48c7ba72adacd240b84a": {"model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_c76e58b8968747bc915fc37594da12fb", "placeholder": "\u200b", "style": "IPY_MODEL_4469976e9ed0411eb472ce953b2b942f", "value": "1/|/100% 1/1 [00:05&lt;00:00,  5.29s/ file]"}}, "e44536ead67141a58ce0aaad0ede93d2": {"model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "e492e498b2da4c3c8be865af31caeac8": {"model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_66669c75d14146139cba84f57a4c12ac", "placeholder": "\u200b", "style": "IPY_MODEL_f6ca90f14db146c0a44e93674585786d", "value": "1/|/100% 1/1 [00:05&lt;00:00,  1.59s/ url]"}}, "e522f0058696468a979a4232787e8057": {"model_module": "@jupyter-widgets/controls", "model_name": "IntProgressModel", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "IntProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "danger", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_bd7d5597bcc24a5ba6ed233ff6869af3", "max": 51785, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_056d566b9dfc4c12b13b0fcb6075a270", "value": 35326}}, "e65cf9ab4221419e97f74426d0ee292f": {"model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "ea263382231d42b29888fb3d1b16acca": {"model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_6276134a3fa7469faa2cdb9b9841a236", "IPY_MODEL_a74e3d4851ad40a9802864c7e9fa3313"], "layout": "IPY_MODEL_3dbe4b29ee9d467cbeef39946925db10"}}, "eaa4b7905b524f5ea5e2d5c73437e0d5": {"model_module": "@jupyter-widgets/controls", "model_name": "IntProgressModel", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "IntProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "info", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_245051a8a9e746568bc24399fded1713", "max": 1, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_f3d20f2e66ed4f059a3a26ffe703adf3", "value": 1}}, "ed49da325e7146d5bddbd6b2ee6e338b": {"model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_de36871931e84fef867700db40d0126c", "IPY_MODEL_2966a7701dde4f46885083de9c6b1eb0"], "layout": "IPY_MODEL_2b125be6a55f43798366013f4d4ee840"}}, "ed874e7a5d294e28940f3ac0dcc99a96": {"model_module": "@jupyter-widgets/controls", "model_name": "IntProgressModel", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "IntProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "danger", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_5599b46c0a374da4a1bc3566747f395c", "max": 1193, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_ac26758ef636439ea359ddf0d1729743", "value": 0}}, "edf5019ce88945bd955f66e539a68c55": {"model_module": "@jupyter-widgets/controls", "model_name": "IntProgressModel", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "IntProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "Dl Completed...", "description_tooltip": null, "layout": "IPY_MODEL_792f241ea8394e9ca81eeac6909481f8", "max": 1, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_84e50d0a4e824d409249d1c05f13a8fc", "value": 1}}, "f3d20f2e66ed4f059a3a26ffe703adf3": {"model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "f6ca90f14db146c0a44e93674585786d": {"model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": ""}}}
</script>


</html>
